{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MH_Multiclass_84944.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9ae5c29649974e7386b5e3c26a1f2f09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_57a33643ae64439c844f0e3d9fb59d63",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_583221106b3742b0a102fd83a2f8d5e2",
              "IPY_MODEL_77ee2931d8134e968492579bdbd0fb9d"
            ]
          }
        },
        "57a33643ae64439c844f0e3d9fb59d63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "583221106b3742b0a102fd83a2f8d5e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4d69101643684b369a8340bbef73999d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 898823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89981f89c5f748edbc3fede4d17d19b7"
          }
        },
        "77ee2931d8134e968492579bdbd0fb9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_851e534c3bdf4ce3bdeb7f531fb54336",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:01&lt;00:00, 551kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9af9a3235cfd4ca6bdb79f1c9e7eff1a"
          }
        },
        "4d69101643684b369a8340bbef73999d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89981f89c5f748edbc3fede4d17d19b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "851e534c3bdf4ce3bdeb7f531fb54336": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9af9a3235cfd4ca6bdb79f1c9e7eff1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "afdde4196cb143778e8b9f4d1ef37d40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aa712d7328a44440ac85c1797cf16af3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1c1cb7f57f934931be2142142217aaa2",
              "IPY_MODEL_3cb9f38bc4d6447bbc280ceb7019f27d"
            ]
          }
        },
        "aa712d7328a44440ac85c1797cf16af3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c1cb7f57f934931be2142142217aaa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_170d255fc82a4b19b4ebcbd0492efbd4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a74ec4c034834aee99b1045bb90b5f83"
          }
        },
        "3cb9f38bc4d6447bbc280ceb7019f27d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_344a38167c3a4eeb857733da5726cd19",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 1.10MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18c5cf4e64474c548b51cf17f80ae776"
          }
        },
        "170d255fc82a4b19b4ebcbd0492efbd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a74ec4c034834aee99b1045bb90b5f83": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "344a38167c3a4eeb857733da5726cd19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18c5cf4e64474c548b51cf17f80ae776": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b5be7b1d33554fcb931bc347966cfa3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_42b59812ffb149049d0f6d08d619292f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_55f202c1897241c5a7340ce0d86ebcce",
              "IPY_MODEL_64191c9fce2040b8bc80ff2b02f4777d"
            ]
          }
        },
        "42b59812ffb149049d0f6d08d619292f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55f202c1897241c5a7340ce0d86ebcce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9b11a207edd54f08b459f333bec431f7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 481,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 481,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4c9eaa396764531919f86c1e31495ee"
          }
        },
        "64191c9fce2040b8bc80ff2b02f4777d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_30981095494e4769a03b3dd27f9bd69e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 481/481 [00:07&lt;00:00, 66.4B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b01be266aa584e26920d9a6e3cc349f4"
          }
        },
        "9b11a207edd54f08b459f333bec431f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4c9eaa396764531919f86c1e31495ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30981095494e4769a03b3dd27f9bd69e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b01be266aa584e26920d9a6e3cc349f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "21f986fb9bbd40c29f96e39efcf34dc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f657d22dfec841f8b2e2f98af3bac6f3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8fbedf8c3a9f4ba4b73dfee0d09be319",
              "IPY_MODEL_738eebca831a4e06b52abaf88be4034d"
            ]
          }
        },
        "f657d22dfec841f8b2e2f98af3bac6f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8fbedf8c3a9f4ba4b73dfee0d09be319": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f196b9b1809c40be89f16d4ff7b2f1c1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 501200538,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 501200538,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7dd66c696b4443e09ee7a7a1ca36e445"
          }
        },
        "738eebca831a4e06b52abaf88be4034d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5335ae258ca140a9b1388867e33eb919",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 501M/501M [00:07&lt;00:00, 70.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ca8f57679e5045d8b933eba458c07bd2"
          }
        },
        "f196b9b1809c40be89f16d4ff7b2f1c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7dd66c696b4443e09ee7a7a1ca36e445": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5335ae258ca140a9b1388867e33eb919": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ca8f57679e5045d8b933eba458c07bd2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "6yQnx-q9ET1H",
        "outputId": "ed939b34-0ace-4174-fae6-9477611ec4c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 608
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/19/22/aff234f4a841f8999e68a7a94bdd4b60b4cebcfeca5d67d61cd08c9179de/transformers-3.3.1-py3-none-any.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 6.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Collecting tokenizers==0.8.1.rc2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/80/83/8b9fccb9e48eeb575ee19179e2bdde0ee9a1904f97de5f02d19016b8804f/tokenizers-0.8.1rc2-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 25.0MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 53.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 53.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=a327db5c9cdc30689a000c18e5c3b6007ed76eab8c45cd72106139dffa1ef9c7\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc2 transformers-3.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RPz2ld5rEYd_"
      },
      "source": [
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "from pathlib import Path \n",
        "\n",
        "import os\n",
        "\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "\n",
        "import random \n",
        "\n",
        "# fastai\n",
        "from fastai import *\n",
        "from fastai.text import *\n",
        "from fastai.callbacks import *\n",
        "\n",
        "# transformers\n",
        "from transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig\n",
        "\n",
        "from transformers import BertForSequenceClassification, BertTokenizer, BertConfig\n",
        "from transformers import RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig\n",
        "from transformers import XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig\n",
        "from transformers import XLMForSequenceClassification, XLMTokenizer, XLMConfig\n",
        "from transformers import DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bOUgI5rahqM9",
        "outputId": "27187f16-3361-4be7-af3a-803f54d3f467",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5KtGKpfOE7u3",
        "outputId": "1c9a8f3a-8561-4e05-9a8b-1e07c5c8c35b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "train_df = pd.read_json(\"/content/drive/My Drive/embold_train.json\").reset_index(drop=True)\n",
        "train_df.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>y-zoom piano roll</td>\n",
              "      <td>a y-zoom on the piano roll would be useful.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>buggy behavior in selection</td>\n",
              "      <td>! screenshot from 2016-02-23 21 27 40  https:/...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>auto update feature</td>\n",
              "      <td>hi,\\r \\r great job so far, @saenzramiro ! : \\r...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>filter out noisy endpoints in logs</td>\n",
              "      <td>i think we should stop logging requests to:\\r ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>enable pid on / pid off alarm actions for ardu...</td>\n",
              "      <td>expected behavior\\r alarm actions pid on and p...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title  ... label\n",
              "0                                  y-zoom piano roll  ...     1\n",
              "1                        buggy behavior in selection  ...     0\n",
              "2                                auto update feature  ...     1\n",
              "3                 filter out noisy endpoints in logs  ...     1\n",
              "4  enable pid on / pid off alarm actions for ardu...  ...     0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cezM6uwBGWdS",
        "outputId": "de04817b-9b14-482c-e00c-8156ec70f1e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "test_df = pd.read_json(\"/content/drive/My Drive/embold_test.json\").reset_index(drop=True)\n",
        "test_df.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>config question  path-specific environment var...</td>\n",
              "      <td>issue description or question\\r \\r hey @artemg...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>crash indien vol</td>\n",
              "      <td>de simulator crasht als hij vol zit</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>unable to mine rocks</td>\n",
              "      <td>sarkasmo starting today, when i hit enter  act...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>not all whitelists are processed</td>\n",
              "      <td>create following rules... order of creation is...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>add ctx menu for idafree 70 and idafree 5</td>\n",
              "      <td>associated with .dll, .dll_, .exe, .exe_, .sc,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title                                               body\n",
              "0  config question  path-specific environment var...  issue description or question\\r \\r hey @artemg...\n",
              "1                                   crash indien vol                de simulator crasht als hij vol zit\n",
              "2                               unable to mine rocks  sarkasmo starting today, when i hit enter  act...\n",
              "3                   not all whitelists are processed  create following rules... order of creation is...\n",
              "4          add ctx menu for idafree 70 and idafree 5  associated with .dll, .dll_, .exe, .exe_, .sc,..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YmcWYJBOGedE",
        "outputId": "41c3c99f-d03c-4ee3-aef6-6697ecfc434e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "train_ex_df = pd.read_json(\"/content/drive/My Drive/embold_train_extra.json\")\n",
        "train_ex_df.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>use a 8bit typeface</td>\n",
              "      <td>since this is meant to emulate some old arcade...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>implement wireless m-bus binding</td>\n",
              "      <td>_from  chris.pa...@googlemail.com  https://cod...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>add multilang support for timeago.js</td>\n",
              "      <td>currently it is only  en . \\r required to add ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>scaleway - seg-fault on shutdown</td>\n",
              "      <td>tbr  irc  creates a new scaleway instance with...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>sistema de pintura: no se guardar los nuevos p...</td>\n",
              "      <td>este sp ya estaba asignado a un carro y se enc...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title  ... label\n",
              "0                                use a 8bit typeface  ...     1\n",
              "1                   implement wireless m-bus binding  ...     1\n",
              "2               add multilang support for timeago.js  ...     1\n",
              "3                   scaleway - seg-fault on shutdown  ...     0\n",
              "4  sistema de pintura: no se guardar los nuevos p...  ...     0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTwS6tl0GlLh",
        "outputId": "bfa0ee26-5888-44a1-de4c-030f704eb3ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "train_df['body']=train_df['title']+\" \"+train_df['body']\n",
        "train_ex_df['body']=train_ex_df['title']+\" \"+train_ex_df['body']\n",
        "\n",
        "train_df1=pd.concat([train_df,train_ex_df],axis=0)\n",
        "train_df1.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>body</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>y-zoom piano roll</td>\n",
              "      <td>y-zoom piano roll a y-zoom on the piano roll w...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>buggy behavior in selection</td>\n",
              "      <td>buggy behavior in selection ! screenshot from ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>auto update feature</td>\n",
              "      <td>auto update feature hi,\\r \\r great job so far,...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>filter out noisy endpoints in logs</td>\n",
              "      <td>filter out noisy endpoints in logs i think we ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>enable pid on / pid off alarm actions for ardu...</td>\n",
              "      <td>enable pid on / pid off alarm actions for ardu...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title  ... label\n",
              "0                                  y-zoom piano roll  ...     1\n",
              "1                        buggy behavior in selection  ...     0\n",
              "2                                auto update feature  ...     1\n",
              "3                 filter out noisy endpoints in logs  ...     1\n",
              "4  enable pid on / pid off alarm actions for ardu...  ...     0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QmmRzcaFGwPM",
        "outputId": "c2cbed3c-3b07-46f0-eefa-fc4be110668d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "import fastai\n",
        "import transformers\n",
        "print('fastai version :', fastai.__version__)\n",
        "print('transformers version :', transformers.__version__)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fastai version : 1.0.61\n",
            "transformers version : 3.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HlheMtIxG4Ly"
      },
      "source": [
        "MODEL_CLASSES = {\n",
        "    'bert': (BertForSequenceClassification, BertTokenizer, BertConfig),\n",
        "    'xlnet': (XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig),\n",
        "    'xlm': (XLMForSequenceClassification, XLMTokenizer, XLMConfig),\n",
        "    'roberta': (RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig),\n",
        "    'distilbert': (DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig)\n",
        "}"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8I02YRgkHD-5"
      },
      "source": [
        "# Parameters\n",
        "seed = 202020\n",
        "use_fp16 = False\n",
        "bs = 16\n",
        "\n",
        "model_type = 'roberta'\n",
        "pretrained_model_name = 'roberta-base'"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qBBNohtQHOXq"
      },
      "source": [
        "model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KBv1XPknHszE"
      },
      "source": [
        "def seed_all(seed_value):\n",
        "    random.seed(seed_value) # Python\n",
        "    np.random.seed(seed_value) # cpu vars\n",
        "    torch.manual_seed(seed_value) # cpu  vars\n",
        "    \n",
        "    if torch.cuda.is_available(): \n",
        "        torch.cuda.manual_seed(seed_value)\n",
        "        torch.cuda.manual_seed_all(seed_value) # gpu vars\n",
        "        torch.backends.cudnn.deterministic = True  #needed\n",
        "        torch.backends.cudnn.benchmark = False\n",
        "seed_all(seed)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jJo0rilIHswU"
      },
      "source": [
        "class TransformersBaseTokenizer(BaseTokenizer):\n",
        "    \"\"\"Wrapper around PreTrainedTokenizer to be compatible with fast.ai\"\"\"\n",
        "    def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):\n",
        "        self._pretrained_tokenizer = pretrained_tokenizer\n",
        "        self.max_seq_len = pretrained_tokenizer.max_len\n",
        "        self.model_type = model_type\n",
        "\n",
        "    def __call__(self, *args, **kwargs): \n",
        "        return self\n",
        "\n",
        "    def tokenizer(self, t:str) -> List[str]:\n",
        "        \"\"\"Limits the maximum sequence length and add the spesial tokens\"\"\"\n",
        "        CLS = self._pretrained_tokenizer.cls_token\n",
        "        SEP = self._pretrained_tokenizer.sep_token\n",
        "        if self.model_type in ['roberta']:\n",
        "            tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]\n",
        "            tokens = [CLS] + tokens + [SEP]\n",
        "        else:\n",
        "            tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]\n",
        "            if self.model_type in ['xlnet']:\n",
        "                tokens = tokens + [SEP] +  [CLS]\n",
        "            else:\n",
        "                tokens = [CLS] + tokens + [SEP]\n",
        "        return tokens"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8IRcytHIAP_",
        "outputId": "6208499e-b871-4991-e685-9b9555985502",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "9ae5c29649974e7386b5e3c26a1f2f09",
            "57a33643ae64439c844f0e3d9fb59d63",
            "583221106b3742b0a102fd83a2f8d5e2",
            "77ee2931d8134e968492579bdbd0fb9d",
            "4d69101643684b369a8340bbef73999d",
            "89981f89c5f748edbc3fede4d17d19b7",
            "851e534c3bdf4ce3bdeb7f531fb54336",
            "9af9a3235cfd4ca6bdb79f1c9e7eff1a",
            "afdde4196cb143778e8b9f4d1ef37d40",
            "aa712d7328a44440ac85c1797cf16af3",
            "1c1cb7f57f934931be2142142217aaa2",
            "3cb9f38bc4d6447bbc280ceb7019f27d",
            "170d255fc82a4b19b4ebcbd0492efbd4",
            "a74ec4c034834aee99b1045bb90b5f83",
            "344a38167c3a4eeb857733da5726cd19",
            "18c5cf4e64474c548b51cf17f80ae776"
          ]
        }
      },
      "source": [
        "transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)\n",
        "transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)\n",
        "fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[])"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9ae5c29649974e7386b5e3c26a1f2f09",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "afdde4196cb143778e8b9f4d1ef37d40",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:1324: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArBvUsphIBxI"
      },
      "source": [
        "class TransformersVocab(Vocab):\n",
        "    def __init__(self, tokenizer: PreTrainedTokenizer):\n",
        "        super(TransformersVocab, self).__init__(itos = [])\n",
        "        self.tokenizer = tokenizer\n",
        "    \n",
        "    def numericalize(self, t:Collection[str]) -> List[int]:\n",
        "        \"Convert a list of tokens `t` to their ids.\"\n",
        "        return self.tokenizer.convert_tokens_to_ids(t)\n",
        "        #return self.tokenizer.encode(t)\n",
        "\n",
        "    def textify(self, nums:Collection[int], sep=' ') -> List[str]:\n",
        "        \"Convert a list of `nums` to their tokens.\"\n",
        "        nums = np.array(nums).tolist()\n",
        "        return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)\n",
        "    \n",
        "    def __getstate__(self):\n",
        "        return {'itos':self.itos, 'tokenizer':self.tokenizer}\n",
        "\n",
        "    def __setstate__(self, state:dict):\n",
        "        self.itos = state['itos']\n",
        "        self.tokenizer = state['tokenizer']\n",
        "        self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)})"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WXNsRL7kIB3l"
      },
      "source": [
        "transformer_vocab =  TransformersVocab(tokenizer = transformer_tokenizer)\n",
        "numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)\n",
        "\n",
        "tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer, include_bos=False, include_eos=False)\n",
        "\n",
        "transformer_processor = [tokenize_processor, numericalize_processor]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Eya6MkFIBub"
      },
      "source": [
        "pad_first = bool(model_type in ['xlnet'])\n",
        "pad_idx = transformer_tokenizer.pad_token_id"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lLfcUPZaIcfR",
        "outputId": "a09cd354-b3bf-41eb-9264-80ff40d9a888",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "tokens = transformer_tokenizer.tokenize('Salut c est moi, Hello it s me')\n",
        "print(tokens)\n",
        "ids = transformer_tokenizer.convert_tokens_to_ids(tokens)\n",
        "print(ids)\n",
        "transformer_tokenizer.convert_ids_to_tokens(ids)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Sal', 'ut', 'Ġc', 'Ġest', 'Ġmo', 'i', ',', 'ĠHello', 'Ġit', 'Ġs', 'Ġme']\n",
            "[18111, 1182, 740, 3304, 7458, 118, 6, 20920, 24, 579, 162]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Sal', 'ut', 'Ġc', 'Ġest', 'Ġmo', 'i', ',', 'ĠHello', 'Ġit', 'Ġs', 'Ġme']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kYhULwS4IcnB",
        "outputId": "38d06625-d8b5-461b-ec9d-6da0f6806ec8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "databunch = (TextList.from_df(train_df1, cols='body', processor=transformer_processor)\n",
        "             .split_by_rand_pct(0.05,seed=seed)\n",
        "             .label_from_df(cols= 'label')\n",
        "             .add_test(test_df)\n",
        "             .databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I136s9ZzIunS"
      },
      "source": [
        "# defining our model architecture \n",
        "class CustomTransformerModel(nn.Module):\n",
        "    def __init__(self, transformer_model: PreTrainedModel):\n",
        "        super(CustomTransformerModel,self).__init__()\n",
        "        self.transformer = transformer_model\n",
        "        \n",
        "    def forward(self, input_ids, attention_mask=None):\n",
        "        \n",
        "        # attention_mask\n",
        "        # Mask to avoid performing attention on padding token indices.\n",
        "        # Mask values selected in ``[0, 1]``:\n",
        "        # ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.\n",
        "        attention_mask = (input_ids!=pad_idx).type(input_ids.type()) \n",
        "        \n",
        "        logits = self.transformer(input_ids,\n",
        "                                  attention_mask = attention_mask)[0]   \n",
        "        return logits\n"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4K3HVNa4I7GK",
        "outputId": "da6e21eb-01f1-4350-fd43-d2b856cde8c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 620,
          "referenced_widgets": [
            "b5be7b1d33554fcb931bc347966cfa3f",
            "42b59812ffb149049d0f6d08d619292f",
            "55f202c1897241c5a7340ce0d86ebcce",
            "64191c9fce2040b8bc80ff2b02f4777d",
            "9b11a207edd54f08b459f333bec431f7",
            "a4c9eaa396764531919f86c1e31495ee",
            "30981095494e4769a03b3dd27f9bd69e",
            "b01be266aa584e26920d9a6e3cc349f4"
          ]
        }
      },
      "source": [
        "config = config_class.from_pretrained(pretrained_model_name)\n",
        "config.num_labels = 3\n",
        "config.use_bfloat16 = use_fp16\n",
        "print(config)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b5be7b1d33554fcb931bc347966cfa3f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kdRIljYhI7Mi",
        "outputId": "1d92c110-be1e-4061-af1a-6c06dd46a125",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170,
          "referenced_widgets": [
            "21f986fb9bbd40c29f96e39efcf34dc4",
            "f657d22dfec841f8b2e2f98af3bac6f3",
            "8fbedf8c3a9f4ba4b73dfee0d09be319",
            "738eebca831a4e06b52abaf88be4034d",
            "f196b9b1809c40be89f16d4ff7b2f1c1",
            "7dd66c696b4443e09ee7a7a1ca36e445",
            "5335ae258ca140a9b1388867e33eb919",
            "ca8f57679e5045d8b933eba458c07bd2"
          ]
        }
      },
      "source": [
        "transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)\n",
        "\n",
        "custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model)\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "21f986fb9bbd40c29f96e39efcf34dc4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=501200538.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loo-WYhMI7DY"
      },
      "source": [
        "from fastai.callbacks import *\n",
        "from transformers import AdamW\n",
        "from functools import partial\n",
        "\n",
        "CustomAdamW = partial(AdamW, correct_bias=False)\n",
        "\n",
        "learner = Learner(databunch, \n",
        "                  custom_transformer_model, \n",
        "                  opt_func = CustomAdamW, \n",
        "                  metrics=[accuracy, error_rate])\n",
        "\n",
        "# Show graph of learner stats and metrics after each epoch.\n",
        "learner.callbacks.append(ShowGraph(learner))\n",
        "\n",
        "# Put learn in FP16 precision mode. --> Seems to not working\n",
        "if use_fp16: learner = learner.to_fp16()"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGzwjMMiJIFf"
      },
      "source": [
        "# For roberta-base\n",
        "list_layers = [learner.model.transformer.roberta.embeddings,\n",
        "              learner.model.transformer.roberta.encoder.layer[0],\n",
        "              learner.model.transformer.roberta.encoder.layer[1],\n",
        "              learner.model.transformer.roberta.encoder.layer[2],\n",
        "              learner.model.transformer.roberta.encoder.layer[3],\n",
        "              learner.model.transformer.roberta.encoder.layer[4],\n",
        "              learner.model.transformer.roberta.encoder.layer[5],\n",
        "              learner.model.transformer.roberta.encoder.layer[6],\n",
        "              learner.model.transformer.roberta.encoder.layer[7],\n",
        "              learner.model.transformer.roberta.encoder.layer[8],\n",
        "              learner.model.transformer.roberta.encoder.layer[9],\n",
        "              learner.model.transformer.roberta.encoder.layer[10],\n",
        "              learner.model.transformer.roberta.encoder.layer[11],\n",
        "              learner.model.transformer.roberta.pooler]"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lp4k1KT4oioU",
        "outputId": "ba69bce6-d584-4791-e736-836c6b86c21b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "learner.save('untrain')\n",
        "seed_all(seed)\n",
        "learner.load('untrain')\n",
        "learner.freeze_to(-1)\n",
        "learner.summary()\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "CustomTransformerModel\n",
              "======================================================================\n",
              "Layer (type)         Output Shape         Param #    Trainable \n",
              "======================================================================\n",
              "Embedding            [512, 768]           38,603,520 True      \n",
              "______________________________________________________________________\n",
              "Embedding            [512, 768]           394,752    True      \n",
              "______________________________________________________________________\n",
              "Embedding            [512, 768]           768        True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [12, 512, 512]       0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           590,592    True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [512, 3072]          2,362,368  True      \n",
              "______________________________________________________________________\n",
              "Linear               [512, 768]           2,360,064  True      \n",
              "______________________________________________________________________\n",
              "LayerNorm            [512, 768]           1,536      True      \n",
              "______________________________________________________________________\n",
              "Dropout              [512, 768]           0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [768]                590,592    True      \n",
              "______________________________________________________________________\n",
              "Dropout              [768]                0          False     \n",
              "______________________________________________________________________\n",
              "Linear               [3]                  2,307      True      \n",
              "______________________________________________________________________\n",
              "\n",
              "Total params: 124,647,939\n",
              "Total trainable params: 124,647,939\n",
              "Total non-trainable params: 0\n",
              "Optimized with 'transformers.optimization.AdamW', correct_bias=False\n",
              "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
              "Loss function : FlattenedLoss\n",
              "======================================================================\n",
              "Callbacks functions applied \n",
              "    ShowGraph"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AWnUoRNeo2XE",
        "outputId": "17694bd9-ff41-4dfe-b4a7-89770d1c5769",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "learner.lr_find()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/1 00:00<00:00]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>error_rate</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='73' class='' max='25312' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.29% [73/25312 00:53<5:08:22 3.7590]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4QPj02gHo7LT",
        "outputId": "2db6e00c-bbe8-4b49-ad26-08e970a21549",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        }
      },
      "source": [
        "learner.recorder.plot(skip_end=10,suggestion=True)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Min numerical gradient: 1.10E-04\n",
            "Min loss divided by 10: 1.32E-05\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3hc9X3n8fd3pNH9bsuWsSwbHLDNxeZiLiUkMU1LMU1Dtk3apYG22aRut2ma7LLZPG2fDWmz+2y32SRNlhLiTVk2aQptEtomBFJCAiGEBWobX7BljMHYFpas64wuo8tI+u4fc2QLoyvSmYvm83qeeaSZc2bOd36W5zPn/H7nd8zdERGR/BXJdAEiIpJZCgIRkTynIBARyXMKAhGRPKcgEBHJc4WZLmC+li9f7uvWrct0GSIiOWX37t2d7l4/1bKcC4J169axa9euTJchIpJTzOz4dMt0aEhEJM8pCERE8pyCQEQkzykIRETynIJARCTPKQhERPKcgkBEJM8pCEREcsCXHn+Zn77cEcprKwhERLLc+LjzpR8d4blXu0N5fQWBiEiW6x8ZZdyhpiwayusrCEREslw8kQSgulRBICKSl+KDCgIRkbwWC/YIasqKQnl9BYGISJaLDY4A6iMQEclbMfURiIjkN/URiIjkufhgkpJohJJoQSivryAQEclyscRIaHsDoCAQEcl68cEkNaXhjBgCBYGISNaLJZJUhzRiCBQEIiJZL7VHoCAQEclb8cGk+ghERPJZLJEM7WQyCDEIzGyNmT1hZofM7KCZfXyKdT5oZvvN7ICZPWNmW8KqR0QkFw0lxxhMjoU2vQRAYWivDKPAne6+x8wqgd1m9kN3PzRpnWPAu9y9x8y2AzuBa0OsSUQkp/QGJ5NVhXhoKLQgcPdWoDX4vc/MmoHVwKFJ6zwz6SnPAo1h1SMikosmzirO+c5iM1sHXAE8N8NqHwYeneb5O8xsl5nt6ugI51JtIiLZKDYRBLnYRzDBzCqA7wCfcPfeada5kVQQfGqq5e6+0923uvvW+vr68IoVEckyYU84B+H2EWBmUVIh8E13f2iadTYDXwO2u3tXmPWIiOSas4eGcvDMYjMz4G+AZnf/wjTrNAEPAXe4+5GwahERyVWxROpaBGGeWRzmHsHbgTuAA2a2N3jsT4AmAHe/F/g0sAy4J5UbjLr71hBrEhHJKfHBJBGDyuLwPq7DHDX0NGCzrPMR4CNh1SAikuvig0mqSqNEIjN+nC6IziwWEclisUS48wyBgkBEJKvFBpNUh3hWMSgIRESyWjzki9KAgkBEJKuFPQU1KAhERLJabDDcmUdBQSAikrXGxz30axGAgkBEJGv1DY/iHu70EqAgEBHJWvHExIRzGjUkIpKXYoPB9BLaIxARyU+xRPhTUIOCQEQka6XjojSgIBARyVoTF6UJc+ZRUBCIiGSteEJ9BCIieS0+mKQ0WkBxYUGo21EQiIhkqVgi/LOKQUEgIpK1Ymk4qxgUBCIiWSsd00uAgkBEJGvFdWhIRCS/xQbDvxYBKAhERLJWqrM43HmGQEEgIpKVhpJjDI+O5/YegZmtMbMnzOyQmR00s49PsY6Z2ZfN7KiZ7TezK8OqR0Qkl0xML5GOICgM8bVHgTvdfY+ZVQK7zeyH7n5o0jrbgQuD27XAV4KfIiJ5LV0TzkGIewTu3urue4Lf+4BmYPU5q90KfN1TngVqzGxVWDWJiOSKsxPOLZE+AjNbB1wBPHfOotXAyUn3W3hzWGBmO8xsl5nt6ujoCKtMEZGsEQvmGcrpPYIJZlYBfAf4hLv3vpXXcPed7r7V3bfW19cvboEiIlkolsY+glCDwMyipELgm+7+0BSrvA6smXS/MXhMRCSv9aZpCmoId9SQAX8DNLv7F6ZZ7bvAbwWjh64D4u7eGlZNIiK5IpZIUhAxKovDHNOTEuYW3g7cARwws73BY38CNAG4+73AI8AtwFEgAXwoxHpERHJGbHCEqpJCUt+pwxVaELj708CM78DdHfhoWDWIiOSqdJ1VDDqzWEQkK6Vr5lFQEIiIZCUFgYhInkvX1clAQSAikpXig0lqtEcgIpKfxsad3qEk1eosFhHJT31DSdzTc1YxKAhERLLO2QnnFAQiInkpnVNQg4JARCTrpHPCOVAQiIhknXROQQ0KAhGRrHNm5tE0XJQGFAQiIllnoo9Ah4ZERPJUbDBJWVEBRYXp+YhWEIiIZJl0nlUMCgIRkawTSySpUhCIiOSv+OBI2kYMgYJARCTrxBJJatI0YggUBCIiWSc+mL4pqEFBICKSVdydWBovSgMKAhGRrDKUHGdkdJzqpbBHYGb3mVm7mb04zfJqM/ueme0zs4Nm9qGwahERyRVnZx5dGn0E9wM3z7D8o8Ahd98CbAM+b2bpe+ciIlkoNpiaZ2hJHBpy96eA7plWASrNzICKYN3RsOoREckF6Z6CGjLbR3A3sAk4BRwAPu7u41OtaGY7zGyXme3q6OhIZ40iImkVT/MU1JDZIPglYC9wHnA5cLeZVU21orvvdPet7r61vr4+nTWKiKRVPM0TzkFmg+BDwEOechQ4BmzMYD0iIhk30UeQL4eGTgDvBjCzlcAG4NUM1iMiknGxRJKCiFFRXJi2bYa2JTN7gNRooOVm1gLcBUQB3P1e4LPA/WZ2ADDgU+7eGVY9IiK5YGLm0dQ4mvQILQjc/bZZlp8Cbgpr+yIiuSjdZxWDziwWEckqr7T301BdktZtKghERLLEqx39HG7r4xc2rUzrdhUEIiJZ4tEX2wDYfllDWrerIBARyRKPHGjlyqYaVlWXpnW7CgIRkSxwvGuAg6d6ueWyVWnftoJARCQLTBwWuvnS9B4WAgWBiEhWePRAK1saq2msLUv7thUEIiIZdrI7wb6WONszcFgI5hgEZlZuZpHg94vM7L1mlt4zHkRElqgfBIeFbrk0i4MAeAooMbPVwGPAHaQuPCMiIgv0yIutXLq6iqZl6T8sBHMPAnP3BPCrwD3u/gHgkvDKEhHJD6dig7xwIsb2DO0NwDyCwMx+Dvgg8P3gsYJwShIRyR8Th4W2Z2C00IS5BsEngD8G/tHdD5rZBcAT4ZUlIpIfHjnQysaGSi6or8hYDXOafdTdfwL8BCDoNO509z8KszARkaWuLT7EruM93PmLF2W0jrmOGvo7M6sys3LgReCQmX0y3NJERJa2fzk4MbdQ5voHYO6Hhi52917gfcCjwPmkRg6JiMhb9MiBVi5aWcHbVmTusBDMPQiiwXkD7wO+6+5JwMMrS0RkaevoG+b517ozOlpowlyD4KvAa0A58JSZrQV6wypKRGSpe7z5NO7pn3J6KnPtLP4y8OVJDx03sxvDKUlEZOnb3xKjtizKhpWVmS5lzp3F1Wb2BTPbFdw+T2rvQERE3oLm1j42NlSl9SL105nroaH7gD7g14NbL/B/ZnqCmd1nZu1m9uIM62wzs71mdtDMfjLXokVEctnYuPNSWx8bV2V+bwDmeGgIWO/uvzbp/p+Z2d5ZnnM/cDfw9akWmlkNcA9ws7ufMLMVc6xFRCSnnehOMJgcY9OqqkyXAsx9j2DQzG6YuGNmbwcGZ3qCuz8FdM+wym8CD7n7iWD99jnWIiKS0w63psbabGrIjiCY6x7B7wNfN7Pq4H4P8NsL3PZFpIalPglUAl9y9yn3HkRElpLm1l4iBheuzOz5AxPmOmpoH7DFzKqC+71m9glg/wK3fRXwbqAU+H9m9qy7Hzl3RTPbAewAaGpqWsAmRUQyr7mtjwvqKyiJZsfcnfO6Qpm79wZnGAP8xwVuuwX4F3cfcPdOUtc82DLNdne6+1Z331pfX7/AzYqIZNbhtl42NmRHRzEs7FKVCx3z9M/ADWZWaGZlwLVA8wJfU0Qkq/UNJTnZPZg1HcUw9z6Cqcw4xYSZPQBsA5abWQtwFxAFcPd73b3ZzH5A6vDSOPA1d592qKmIyFLwUlsfAJuyZOgozBIEZtbH1B/4Ruq4/rTc/bbZNu7unwM+N9t6IiJLRXMQBBuzZMQQzBIE7p49kSUisgQ0t/ZSVVLIquqSTJdyxkL6CEREZJ4Ot/ayaVV2TC0xQUEgIpIm48HUEtnUUQwKAhGRtDnZk2BgZCyrho6CgkBEJG2aWydGDGmPQEQkLx1u68UMLsqCaxBMpiAQEUmT5tZezl9WTmlRdkwtMUFBICKSJoezsKMYFAQiImkxMDzK8a5E1nUUg4JARCQtDk+cUaw9AhGR/HS4LTVxs/YIRETy1OHWPiqLC2msnXGatoxQEIiIpEFzay8bV1Vm1dQSExQEIiIhc3cOt/Vl1YyjkykIRERC1tIzSP/waFYOHQUFgYhI6Jpbg47iLLoYzWQKAhGRkB1u68MMNmTZ1BITFAQiIiE73NbL2royyosXcnXg8CgIRERC1tyavR3FoCAQEQlVYmSU17oGsrZ/ABQEIiKhOnSqF/fsuwbBZKEFgZndZ2btZvbiLOtdbWajZvb+sGoREcmUnx3twgyuWVeX6VKmFeYewf3AzTOtYGYFwP8AHguxDhGRjHn6aAeXra6mtrwo06VMK7QgcPengO5ZVvsY8B2gPaw6REQypX94lBdOxHj725ZnupQZZayPwMxWA/8G+Moc1t1hZrvMbFdHR0f4xYmILILnj3UxOu68Q0Ewrb8CPuXu47Ot6O473X2ru2+tr69PQ2kiIgv305c7KS6McOXa2kyXMqNMnt2wFXgwmIlvOXCLmY26+z9lsCYRkUXzs6OdXHN+HSXR7LpG8bkyFgTufv7E72Z2P/CwQkBElor23iGOnO7n165szHQpswotCMzsAWAbsNzMWoC7gCiAu98b1nZFRLLB00c7AbK+oxhCDAJ3v20e6/5OWHWIiGTC00c7qSsv4uIsPpFsgs4sFhFZZO7Oz452cv36ZUQi2XdFsnMpCEREFtnR9n5O9w5zQw4cFgIFgYjIopvoH7jhQgWBiEhe+tnRTtYtK6OxtizTpcyJgkBEZBElx8Z59tXunBgtNEFBICKyiPadjNE/PMo7cuSwECgIREQW1dNHO4kY/NwFCgIRkbz09MudXNZYQ3VZNNOlzJmCQERkkfQNJXnhZIwb3rYs06XMi4JARGSRPH+sm7Fxz6mOYlAQiIgsmp++3ElJNMJVWT7t9LkUBCIii+Tpo51cc/4yiguze9rpcykIREQWwSsd/Rxt7+fGDbl38SwFgYjIIvj+/lbM4JbLVmW6lHlTEIiILIKH95/i6nV1rKwqyXQp86YgEBFZoCOn+zhyup/3bM69vQFQEIiILNjD+1uJGNx8aUOmS3lLFAQiIgvg7jy8/xTXnr+MFZW5d1gIFAQiIgtyuK2PVzsGeM+W3DwsBAoCEZEFeXj/KQoixs2X5OZhIQgxCMzsPjNrN7MXp1n+QTPbb2YHzOwZM9sSVi0iImFwd76/v5Xr1y9jWUVxpst5y8LcI7gfuHmG5ceAd7n7ZcBngZ0h1iIisugOnurlta5Ezo4WmlAY1gu7+1Nmtm6G5c9Muvss0BhWLSIiYXh4fyuFEeOXcviwEGRPH8GHgUenW2hmO8xsl5nt6ujoSGNZIiJTmxgtdMOFy6kpK8p0OQuS8SAwsxtJBcGnplvH3Xe6+1Z331pfn3vzeIjI0rOvJU5LzyC/nINTSpwrtENDc2Fmm4GvAdvdvSuTtYiIzMf3958iWmDclOOHhSCDQWBmTcBDwB3ufiRTdYiIABxt72fcnZrSKNVl0Rmnkh4fT40WeueF9VSX5s4lKacTWhCY2QPANmC5mbUAdwFRAHe/F/g0sAy4x8wARt19a1j1iIhMZWzc+e+PNPO1p4+94fHSaAE1ZVGWVRSxYWUVm1ZVcvGqKjatquLVzgFOxYf45M0bMlT14gpz1NBtsyz/CPCRsLYvIjKb/uFRPv7AC/zocDu3X9fEdRcsI5ZIEh9MEkuMEEskaesd4qmXO/jOnpYzzyuNFlBUGOEXNq3MYPWLJ6N9BCIimdLSk+Aj/3cXL7f389lbL+GOn1s34/odfcM0t/aeuV18XhWVJbl/WAgUBCKSh3Yf7+H3vrGL4dFx7v/Q1bzjwtlHI9ZXFlNfWc87L1p6IxcVBCKSV7637xR3fmsfq6pLeHDHVt62ojLTJWWcgkBE8kZbfIg7v7WPzaur2flbW6krz+0TwRZLxk8oExFJl3uePMr4uPPF37hcITCJgkBE8sKp2CAPPn+SD2xtZE1dWabLySoKAhHJC1958hUc56M3vi3TpWQdBYGILHmnYoP8/b+e5ANb19BYq72BcykIRGTJ++snjmpvYAZ5P2rI3ekfHqWjbzh16x8+8ztAdWmU6tIoNWVRqkqj1JQWUVdeRG35zHORzLS9zv4RRsbGKSqIUFQYobgwQlFBhEjEFvvtieS9lp4E/7DrJL++dQ2ra0ozXU5WypsgOHgqzkN7Xqd7YISugRG6+ofP/D4yOv6m9QuDD+XRcZ/2NSuKC6krD4KhLEplSZTy4kIqSwopLyqkoqSQ0bFxWnoGOdmT4GR3gpaeQYan2B5AUUGE0qICyosKKCsupLyogPLiQkqjBRREjIKIEYkYBZb6vbKkkKa6Mprqyli7rJymujJKi+YfTiJL2T1PvoJh2huYQd4EQUvPIA88f4JlFUXUlRezsqqETauqWFZRxLLyIlZUlgRnDhZTX1FMTVnq1PHEyBixwSTxRJLY4AjxRJLuxAg9QYh0B7eO/mFe60rQNzRK/3CSoeTZD/vq0iiNtaVcuKKSGzesoLG2lNKiAkZGxxkObhO/D46MMjAyRmJklIHh1M/4YJKxcU/d3BkPfsYSSfqGRt/wPldUFtNUV8aaiVttaSoslpXRUFVCMMGfSF5o6UnwrV0n+bdXN3Ge9gamlTdBcNPFKzn05zNdQnlq5cWFlBcXznuXcnRsnIHhMSwCVSHNR+LuxAeTHO9KcLw7wYmuAY53JTjZk+D5Y938097X8Uk7NJXFhVzUUMmGhko2NlRy0cpKNjVUUV22NOZLETnXXz9xFMP4gxvXZ7qUrJY3QZDub8KFBRGqy8LtizczasqKqCkrYsuamjctHxkd51RskBPdCY53DXDkdD8vtfXx8L5T/N1zZ/ckmurK2NxYzZbGGjY3VnPJ6moqivPmT0OWqJPdCb61q4XfvLaJVdXaG5iJ/rcvYUWFEdYtL2fd8nLg7ERZ7s7p3mEOt/VyqLWXAy1xXjgR4+H9rQCYwbpl5axdVsa6ZeWsW1bG2uXlrFtWzuqaUooKNdhM0mtgeJQfHjrNjw+3Y5Y63FpTmhrAUV0apagwwuneIU7FhjgVG6Q1PsTxrgEiEeMPtqlvYDYKgjxkZjRUl9BQXcK2DSvOPN7ZP8yBljj7WmK8fLqf17oG+Ndj3QyMjJ1ZJ2JwXk3pmU7qpmVlrK0r57LV1aypK1UfhEzJ3ekeGGE06Osad2d8HMbdiRZGqCsretNAh5HRcX5ypIN/3vs6jzefZig5zorKYkqiBcQHk/QOJd9w6BNSAzjOqylhVXUpl66u4qaLG2ioLknjO81NCgI5Y3lFMTduXMGNG8+Gg7vTNTDCa50DvNaV6oc40Z3gRHeCx5vb6ewfnvT8Ii5fU8sVTTVc0VTD21ZU0D80Sk8iSXxwhJ6BJD2JEUqLCs5c6akkqlFOS9mR0308vL+VRw60crS9f8Z1S6MFZ4Zm15QWceD1OPHBJLVlUd5/VSO3Xr6aq5pqzwyzHh93+oZH6R1MMjw6xoqqktD645Y683MjNctt3brVd+3alekyJDAwPMqxzgH2tcR44USMPSd6eLVjYE7PjRisr6/gkvOquOS8atavKGdNbRmNtRoGm01OdCU41BpnRVUJ51WXUl9ZTMEU57yMjTuxxAit8SEebz7N9/e38nJ7P2Zw7fl1/PzGFVQUR4kYRCJGxIyCCAwnx+lJJOkeGKZ7IPiZSHLB8nLeu+U8brhwOdECHY5cKDPbPd3lgBUEsuhiiRFeOBnjRFeCqtJCasqKqC0roqY0Sm1ZEb1DSQ6einPwVG9wi3O6d/gNr7G8opg1daWsqS1j3fJy1teXc37Q36FvfenxWucAdz9xlH984XXGJp1PUxgxVlalDi0a0J1IDaGOD549VGMG16yr45c3r+LmSxtYUanDM5mmIJCs1xWch9ESnHh3sjt1Et6J7gSvxwbfcCx4eUUxGxsquXHjCm66eKVmkpynrv5hvvyjlyktKuTyNdVsbqxhVfXZc0wmB0BhxLj9urX8ypbz6B4Y5lRsiNZ4qjO2NTaEGdSWF1FXVhT8jFJXUcx159exokof/tlEQSA5bSg5xsnuBK92DnCsc4BjHQO8cLKHI6dTx5w3NlTyixev5BcvXsmFKyrpG0p1JPYOjdI3lDqG3N43zOneIVrjQ7TFB2nrHaKzb4TiaCR1FnhxIWXFBVQUF7KisoR3bajnXRfVU126CHsfr7wCn/88/O3fQn8/VFTA7bfDnXfC+vSObz94Ks6Or++mvW8IgORY6v9/fWUxWxqrKYkW8OiLbWcC4PfedYG+zS8RGQkCM7sPeA/Q7u6XTrHcgC8BtwAJ4Hfcfc9sr6sgkAnHuwb44aHTPHboNLte62aG2UAAKIlGaAgOaTRUlbCsopjk2Dj9w6MMDI+SGBmjf3iU410JugdGKIwYV6+r492bVvDuTStprC2lq3+Ejr5hOvtT81J1D4zQWFvKVWtrpx6r/uij8P73QzKZuk2IRlO3b38btm9f3IaZxvf2neKT395HbVkRX73jKjY0VNLc2sf+lhh7T8bY3xKnLT7Eb1y9RgGwBGUqCN4J9ANfnyYIbgE+RioIrgW+5O7Xzva6CgKZSvfACE8cbqe9b5jKkkKqSqOpnyVRqkoKqa8spro0OqfhrWPjzt6TPTze3M6Pm9t56XTfnGpYVV3ClU2pUVNb19Vx2VAnBZdvgURi+ieVlcH+/aHuGYyNO59/7CXuefIVtq6t5Z7br9SHfB7K2KEhM1sHPDxNEHwVeNLdHwjuvwRsc/fWmV5TQSDpdrI7wY8Pt9OTGKG+spjlFanbisrUnFSvdgyw50QPu4/38MKJGK/HBgH4yx9/lV/d8yiFY6PTv3g0Cjt2wN13v6XauvqH2XMixu7jPbT3DdFQVcKq6hIaqktZVV1CdWmUT//zizzxUge3XdPEn733Ep0QmKeyNQgeBv7C3Z8O7v8I+JS7v+lT3sx2ADsAmpqarjp+/HhoNYssVFt8iOeOdXHT9RsoHZzDUNqqKojHp12cHBunq38kdTiqb5iWngQvnIyx53gPr3Wl9jYKI0Z9ZTEdfcNvmjG3MGJ85r2XcPt1axf0viS3zRQEOXFCmbvvBHZCao8gw+WIzKihuoRbL18NQzMcEprE+/vBnVPxIV5q6+VwWx8vtfVx5HQ/p3uH6B4YedNzllcUc2VTDbdd08SVa2u5bHWqo3d83OkcGKYtnuoYP907xBVrarmssXqx36YsIZkMgteBNZPuNwaPiSwNFRXQN3v/Qn+0hOs/8xh9w2cPIZ1XXcJFDZVc2VRzZnr05RWpnxOHf6bq74hEjBWVJayoLGFz46K+G1nCMhkE3wX+0MweJNVZHJ+tf0Akp9x+O3zta28cLXSO8cJCmt/9Xm694jw2NFSdmR58UYatisxRmKOGHgC2AcuB08BdQBTA3e8Nho/eDdxMavjoh6bqHziXOoslZ7zyCmzenPFRQyKQoT4Cd79tluUOfDSs7Ytk3Pr1qfMEZjuPQCEgGaZxZCJh2r499Y1/x47U6KBIJPVzx47U42k6mUxkJppiQkQkD8x0aEh7BCIieU5BICKS5xQEIiJ5TkEgIpLnFAQiInku50YNmVkHEAOmm6WrepplUz1+7mOT75+7bDnQOd96ZzFdrQt9zkzrzLV9ZmuvfGqf+f7tnHs/jLaZrq7FeE6+tk+6/m9N9Vg62metu9dPucTdc+4G7JzvsqkeP/exyfenWLYrne9jIc9ZjPaZrb3yqX3m+7czRVstetuofRa/fdL1fytb2mfyLVcPDX3vLSyb6vFzH/veDMvC8Fa2MZfnLEb7zNZe+dQ+8/3bmWsdC6X2mdl8t5Gu/1tTPZaJ9jkj5w4NZYqZ7fJpTsYQtc9M1DYzU/vMLB3tk6t7BJmwM9MFZDm1z/TUNjNT+8ws9PbRHoGISJ7THoGISJ5TEIiI5Lm8DAIzu8/M2s3sxbfw3KvM7ICZHTWzL9uk6wWa2cfM7LCZHTSzv1zcqtMjjLYxs8+Y2etmtje43bL4ladHWH87wfI7zczNbPniVZxeIf39fNbM9gd/O4+Z2XmLX3l6hNQ+nws+d/ab2T+aWc18XzsvgwC4n9SV0d6KrwC/C1wY3G4GMLMbgVuBLe5+CfA/F15mRtzPIrdN4Ivufnlwe2RhJWbU/YTQPma2BrgJOLHA+jLtfha/fT7n7pvd/XLgYeDTCy0yg+5n8dvnh8Cl7r4ZOAL88XxfOC+DwN2fAronP2Zm683sB2a228x+amYbz32ema0Cqtz9WU/1sn8deF+w+N8Df+Huw8E22sN9F+EIqW2WjBDb54vAfwZyevRGGO3j7r2TVi0nh9sopPZ5zN1Hg1WfBRrnW1deBsE0dgIfc/ergP8E3DPFOquBlkn3W4LHAC4C3mFmz5nZT8zs6lCrTa+Ftg3AHwa7rveZWW14pWbEgtrHzG4FXnf3fWEXmiEL/vsxs/9mZieBD5LbewRTWYz/XxP+HfDofAsI7ZrFucTMKoDrgW9NOmxbPM+XKQTqgOuAq4F/MLMLPMfH5y5S23wF+Cypb3KfBT5P6g825y20fcysDPgTUoeFlpxF+vvB3f8U+FMz+2PgD4G7Fq3IDFqs9gle60+BUeCb832ugiAlAsSCY5BnmFkBsDu4+11SH2iTd7sagdeD31uAh4IP/ufNbJzUZFEdYRaeBgtuG3c/Pel5/5vUcd6lYqHtsx44H9gXfBA0AnvM7Bp3bwu59nRYjP9bk30TeIQlEgQsUvuY2e8A7wHe/Za+fIY9mVG23oB1wIuT7j8DfCD43Uh1+k71vOdJfes3UrtgtwSP/z7w58HvFwEnCU7Yy7VbCG2zatI6/wF4MNPvMZva55x1XgOWZ/o9ZtOxraYAAAM/SURBVFP7ABdOWudjwLcz/R6zrH1uBg4B9W+5pkw3Sob+IR4AWoEkqW/yHyb1rewHwL6gUT89zXO3Ai8CrwB3T3zYA0XA3wbL9gA/n+n3mUVt8w3gALCf1LebVel6P7nQPuesk9NBENLfz3eCx/eTmoxtdabfZ5a1z1FSXzz3Brd751uXppgQEclzGjUkIpLnFAQiInlOQSAikucUBCIieU5BICKS5xQEsiSYWX+at/fMIr3ONjOLBzNrHjazWScrNLP3mdnFi7F9EVAQiEzJzGY8697dr1/Ezf3UU2eWXgG8x8zePsv67wMUBLJoFASyZE03q6OZ/UowOeALZva4ma0MHv+MmX3DzH4GfCO4f5+ZPWlmr5rZH0167f7g57Zg+beDb/TfnDRP/C3BY7uD+eNnnFrD3QdJnRA0MRnd75rZv5rZPjP7jpmVmdn1wHuBzwV7EevnMnulyEwUBLKUTTer49PAde5+BfAgqemfJ1wM/IK73xbc3wj8EnANcJeZRafYzhXAJ4LnXgC83cxKgK8C24Pt189WbDAr64XAU8FDD7n71e6+BWgGPuzuz5A6O/uTnrq2wyszvE+ROdGkc7IkzTKrYyPw98Ec70XAsUlP/W7wzXzC9z11jYlhM2sHVvLG6YABnnf3lmC7e0nNJdMPvOruE6/9ALBjmnLfYWb7SIXAX/nZyeYuNbP/CtQAFcC/zPN9isyJgkCWqilndQz8L+AL7v5dM9sGfGbSsoFz1h2e9PsYU/+fmcs6M/mpu7/HzM4HnjWzf3D3vaSuZvU+d98XzC65bYrnzvQ+ReZEh4ZkSfLUVa2OmdkHACxlS7C4mrNT+P52SCW8BFxgZuuC+78x2xOCvYe/AD4VPFQJtAaHoz44adW+YNls71NkThQEslSUmVnLpNt/JPXh+eHgsMtBUteUhtQewLfMbDfQGUYxweGlPwB+EGynD4jP4an3Au8MAuS/AM8BPwMOT1rnQeCTQWf3eqZ/nyJzotlHRUJiZhXu3h+MIvpr4GV3/2Km6xI5l/YIRMLzu0Hn8UFSh6O+muF6RKakPQIRkTynPQIRkTynIBARyXMKAhGRPKcgEBHJcwoCEZE89/8B5ILyATniWkgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Is4oFNhKo90-",
        "outputId": "f4153389-f012-4833-819b-762f8e34020b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 325
        }
      },
      "source": [
        "learner.fit_one_cycle(1,max_lr=2e-05,moms=(0.8,0.7))  #0.846622\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>error_rate</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.414978</td>\n",
              "      <td>0.406218</td>\n",
              "      <td>0.846622</td>\n",
              "      <td>0.153378</td>\n",
              "      <td>1:35:43</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU1foH8O+bTqghhBo0oQdpQoAoRQSUACpWBMUGXq5eu16vwXYFQUC9+FMvdrl2QbGA0gSk99ADoQSIEGpooaaR8/tjZzezu7Mt2WSS5ft5Hh92Z2ZnzuzGd86c854zopQCERFVfkFmF4CIiPyDAZ2IKEAwoBMRBQgGdCKiAMGATkQUIELMOnBY1ZqqXUJzsw5PRFQprV+//rhSKsZonWkBPaJ2faSmppp1eCKiSklE/nK1zrQmF6a/ExH5l3kB3awDExEFKBNr6AzpRET+ZFobOhGRrwoKCpCVlYXc3Fyzi1LmIiIiEBsbi9DQUK8/Y1pAZwWdiHyVlZWF6tWrIy4uDiJidnHKjFIKJ06cQFZWFuLj473+HNvQiajSyM3NRXR0dEAHcwAQEURHR/t8J8I2dCKqVAI9mFuV5Dw5UpSIKECY2uTCWjoRVSanT5/GBx984PPnBgwYgNOnT5dBieyZWkO/VMSATkSVh6uAXlhY6PZzs2fPRq1atcqqWDampi0WFimEBJtZAiIi76WkpGDPnj3o0KEDQkNDERERgaioKOzYsQO7du3CrbfeigMHDiA3NxdPPfUURo4cCQCIi4tDamoqzp07h/79+6N79+5YuXIlGjVqhBkzZqBKlSp+KZ/pAZ2IqCRG/7YN2w+d8es+WzesgX/ffJXL9RMmTEBaWho2bdqExYsXY+DAgUhLS7OlFk6ZMgW1a9fGxYsX0blzZ9xxxx2Ijo6228fu3bvx/fff49NPP8XgwYPx008/YdiwYX4pv6kB/dIlBnQiqry6dOlilyf+3nvv4ZdffgEAHDhwALt373YK6PHx8ejQoQMAoFOnTsjMzPRbeUwN6AVFRWYenogqMXc16fJStWpV2+vFixdjwYIFWLVqFSIjI9GrVy/DPPLw8HDb6+DgYFy8eNFv5TG1U7TgEgM6EVUe1atXx9mzZw3X5eTkICoqCpGRkdixYwdWr15dzqUzuYaeV8CATkSVR3R0NLp164Y2bdqgSpUqqFevnm1dcnIyPvroIyQkJKBly5ZISkoq9/KJp1xwEZkC4CYAx5RSbQzWC4B3AQwAcAHAg0qpDZ4OHN6gudq6aQNa1KteooIT0eUnPT0dCQkJZhej3Bidr4isV0olGm3vTZPLFwCS3azvD6C59t9IAB96VVIA+YWsoRMR+YvHgK6UWgrgpJtNBgH4SlmsBlBLRBp4c/A8BnQiIr/xR6doIwAHdO+ztGVORGSkiKSKSCrAGjoRkT+Va5aLUuoTpVSitf0nn1kuRER+44+AfhBAY937WG2ZR6yhExH5jz8C+kwA94tFEoAcpdRhbz7IgE5E5D8eA7qIfA9gFYCWIpIlIiNE5BEReUTbZDaAvQAyAHwK4B/eHvzkhfwSFJmIqHKoVq0aAODQoUO48847Dbfp1asXUlNT/XI8jwOLlFJDPaxXAB4rycFf+TUN9yVdWZKPEhFVGg0bNsT06dPL/Dh8YhERkZdSUlIwefJk2/vXXnsNY8eORZ8+fdCxY0e0bdsWM2bMcPpcZmYm2rSxjMu8ePEihgwZgoSEBNx2221+ncvF1KH/REQlNicFOLLVv/us3xboP8Hl6rvvvhtPP/00HnvM0ijxww8/YN68eXjyySdRo0YNHD9+HElJSbjllltcPhP0ww8/RGRkJNLT07FlyxZ07NjRb8VnQCci8tLVV1+NY8eO4dChQ8jOzkZUVBTq16+PZ555BkuXLkVQUBAOHjyIo0ePon79+ob7WLp0KZ588kkAQLt27dCuXTu/lY8BnYgqJzc16bJ01113Yfr06Thy5AjuvvtufPvtt8jOzsb69esRGhqKuLg4w2lzywPb0ImIfHD33Xdj6tSpmD59Ou666y7k5OSgbt26CA0NxaJFi/DXX3+5/XzPnj3x3XffAQDS0tKwZcsWv5WNNXQiIh9cddVVOHv2LBo1aoQGDRrg3nvvxc0334y2bdsiMTERrVq1cvv5Rx99FA899BASEhKQkJCATp06+a1sDOhERD7aurW4M7ZOnTpYtWqV4Xbnzp0DYHlIdFpaGgCgSpUqmDp1apmUy7Qml1qRoahTLdzzhkRE5BXTAnqwCB9BR0TkR6YF9CARXCy4ZNbhiaiS8vSUtUBRkvM0NaDnFxbhUtHl8eMQUelFRETgxIkTAR/UlVI4ceIEIiIifPqcaZ2ipy/mozaAN+ftwKj+l88zAomo5GJjY5GVlYXs7Gyzi1LmIiIiEBsb69NnTAvo1sfPfbxkLwM6EXklNDQU8fHxZhejwjKtyaV6ODMmiYj8ybSAXrcGUxaJiPzJtIAeGcYaOhGRP3EuFyKiAMGATkQUIBjQiYgCRIUI6IWcAoCIqNRMDegPdYsDAOQzoBMRlZqpAb1xVCQAIL+QAZ2IqLRMDegL0o8CALYfOmNmMYiIAoKpAX3HkbMAgIzsc2YWg4goIJga0N+4rQ0AIKFBDTOLQUQUEEwN6FGRYQDYhk5E5A+mBvTw0GAADOhERP5gakAPC7YcPq+QTy4iIiotk2vo1oDOGjoRUWmZG9BDGNCJiPzFq4AuIskislNEMkQkxWD9FSKySEQ2isgWERngzX7DGNCJiPzGY0AXkWAAkwH0B9AawFARae2w2csAflBKXQ1gCIAPvDl4eIilUzSvgG3oRESl5U0NvQuADKXUXqVUPoCpAAY5bKMAWJPJawI45M3BrU0unMuFiKj0vAnojQAc0L3P0pbpvQZgmIhkAZgN4AmjHYnISBFJFZHU7OxsW5bL7K2HfS03ERE58Fen6FAAXyilYgEMAPC1iDjtWyn1iVIqUSmVGBMTg6AgAQCkHeRcLkREpeVNQD8IoLHufay2TG8EgB8AQCm1CkAEgDr+KCAREXnHm4C+DkBzEYkXkTBYOj1nOmyzH0AfABCRBFgCerY/C0pERO55DOhKqUIAjwOYByAdlmyWbSIyRkRu0TZ7DsDfRGQzgO8BPKiUUmVVaCIichbizUZKqdmwdHbql72qe70dQLeSFOC6FjE4fSG/JB8lIiId058pGhYSxIFFRER+YH5ADw7CKdbQiYhKzasml7I0S8tBP59XiKrhpheHiKjSMr2GbsVmFyKi0jE9oEdoU+jyIRdERKVjekB/47a2AIBcTtBFRFQqpgd064yLuXxqERFRqZge0AuLLE0tJ84x04WIqDRMD+jWtvPME+dNLgkRUeVmekBPaGCZRj2mWrjJJSEiqtxMD+jWLJdcZrkQEZWK6QHd1inKLBciolIxP6BrNfQdh8+aXBIiosrN9IAeEWqpoU9ZsQ9Ld3EKdSKikjI/oGtNLgCwaOcxE0tCRFS5mR7QQ4PF9rrgEjtGiYhKyvSALlIc0Hs0jzGxJERElZvpAV2PNXQiopKrUAE9t4ABnYiopCpUQL/IXHQiohKrUAE9N58BnYiopCpUQB83O93sIhARVVoVKqATEVHJMaATEQWIChHQvxreBQDQo3kdk0tCRFR5VYiA3rNFDNrF1kRwkHjemIiIDFWIgA4AIUGCwkvK7GIQEVVaIWYXwOromTwcPH3a7GIQEVVaFaaGfvD0RbOLQERUqVWYgG61JYu1dCKikvAqoItIsojsFJEMEUlxsc1gEdkuIttE5LuSFmjwx6sQlzKrpB8nIrpseWxDF5FgAJMB3AAgC8A6EZmplNqu26Y5gFEAuimlTolI3ZIWiBN0ERGVjDc19C4AMpRSe5VS+QCmAhjksM3fAExWSp0CAKVUqR89dPxcXml3QUR0WfEmoDcCcED3PktbptcCQAsRWSEiq0Uk2deCDE6MtXt/kRN1ERH5xF+doiEAmgPoBWAogE9FpJbjRiIyUkRSRSQ1O9v+gdD92zbwU1GIiC5P3gT0gwAa697Hasv0sgDMVEoVKKX2AdgFS4C3o5T6RCmVqJRKjImxf9xc20Y17d7nFbItnYjIF94E9HUAmotIvIiEARgCYKbDNr/CUjuHiNSBpQlmb2kKNnOT4zWDiIjc8RjQlVKFAB4HMA9AOoAflFLbRGSMiNyibTYPwAkR2Q5gEYDnlVInfCmI47D/9/7MwK6jZ33ZBRHRZU2UMmf+lMTERJWammp7r5RC/KjZTttNGtweN7dviNDgCjcGioio3InIeqVUotG6ChMlRYxnWnz2h80Y/sW6ci4NEVHlU2ECOgBsH9MPE+9o67R82e7jJpSGiKhyqVABPTIsBLd3jPW8IREROalQAR2Ay7byd+bvwoX8wnIuDRFR5VHhAror7y7cjYHvLTe7GEREFValCegAsO/4ebOLQERUYVWqgG7k1Pl8vDZzG8xKvyQiqigqZEDfOTYZTWKqGq57aupGu/dXvz4fX6zMxLsLd2P+9qPYfMDcB2Scyys05SEdR8/kIudiAQBgT/a5cj8+EZmvQgb08JBg/PlcL8N1MzYdMlz+fwt2429fpWLQ5BUAgJwLBfh6VWa519yf/H4jbvnvCpzPc+7AXf/XKVvQLY2MY/YB+825O9D1jYXoPvFPvPJrGvr8ZwniUmZh2e5sF3uoGC4VKWSf5TTJRP5SIQO6N9buO2n3ZKOQIPuBSU9N24hXZmzDpnKusf+5wzIV/HmHjJzCS0W448OVaD/6Dzz85boSX2jemb8LfSctwTvzd9mWfbZsHwDgbG4hvl79l235fZ+vtR27vDOEMo6dxaT5u9ye55vzdqDzuAU4wbnvifyiQgf0KQ8m4um+TpM2ArA8qk7v+X4t7d4v3mmpnd72wUpMmLOjbAroRp7Dk5cKi4oD24L0Y8i/VLLZJN9duNvuXwBQcB00Cy4V4elpm9D61XklOl5J3ff5Wry3cDdOXXB9RzI37QgA4Ewu01GJ/KFCB/Tereqh4xVRTsuNan3r/zrlcj8fLdnjtCz98BlbQAGAIzm5WO7DiNQlu7KdmjT26tqu8wotD+g4cPIC1u47iSKHMp/1YxAruOQ6oP914gJ+33K4RPvNPH7eqXnH+zJZLlgXC1w/qOSvExcAON9dEVHJVOiADgBX1I50WvbqjG1Oy6xNHQA8PmT65Pl89H93GR75Zj1+22xpk7/9gxUY9vkar8v1wJS1tiaNBduPYueRs+j9nyW29V+szAQA9HhzEQZ/vAqXiuyDbnk9kUn/KD9XzR/7jp/HLxuznJb3ensx+k5aYvAJb46bDwB4e95Op3UFDncnB05eKNExiMhehQ/ocXWqYtm/rrdbpm8ntiosct8mffD0RRw/l4eciwXo+Pp82/K5244gLmUWDuXkAoBXtfQXf9lq9/7hr1LR7/+W2i37ZvV+7D9RHKhW7z1pt77gUhGOnc11e5wN+0/h5veX4/SFfI9lcuXomeJjOF5UrMuuf3sxnpm22W75ygz/zJ/jOAXy3LQjaP7SHCzSXYDv+cz7CykRuVbhAzoANK4did+f6F6qfUycswOJYxeg87gFdstnOTRHWGvpSikcyXEOuH//OhXfrdnv1TF7vrXI9vpvX6Xarbv3szXoMm4hjuTkYsamg+jzn8VONejbP1iJrQdzMOzzNXh22iakHcxxOoanzlV9QDdq/li7r/hCY93XsbO5dkG2yM3FcsC7y/C0QyqpnuOTp/7YZmnmmrnZPlspLmUWZvjpoSZxKbOc+k027j+FwzkXvd6HmXcN2w+dweKdpX7OeoUwaf4ujJ+TbnYxLhuVIqADQBuHR9T5yhpA8r14tF1RkcLMzYeQNH4hNu63b5uft+2o3Xt9wPTFYe1ikTR+IZ6augl7ss/b7jJ+33LIru067eAZ/LzxIG5633nqgw37XfcdWMpX3OSyJcv9BaHly3MRlzILGUft282HfroaGcfO4c8d9ud+4lweth8+g18NUkk7x1n6Pm7vaP888SuiLU1oh047B9enpm5ySvcsaRu+Y7/JbR+sxDXj//Tqs79uPIgeby7y212Krwa8twwP/i8wpox+b+FufLykVA8vIx9UmoAOAH0T6pbLcV6ZkYZJWlrgfq2mVnipCJ8tc/7D7PrGQr8dt+XLcwAAj3+30a7t2lWnYZdxC7Ag3X1N7ojughMeUvxz5xZcwrEz9jVxa+ZNnkMb95p9J9F30hIM/6L4LiOv8BL+NX2Ly+Ouy7RcaIIc5rkXiG2fRlJ+tjRnKaXw8Jfr0HfSEszbdsRw25I4aHAhcbRZGxi2OSsHz/+4GfdPWeu34/vCU5Nceflw8R6M4DMJKoVKFdB7NI/xvJEffLtmvy0DI/P4BczbdgSvzEjD2Fmlv3X8+R/XulxXpIAWWlDXc9U/cOxsHj5c7JzBY7eNLqDf+dEqZByztGkPeG8Zuri4GNWtHu5yf9ZmkZYvz8XCHZ6bBXYdsW9DP5PrfmDVb5sP4c8dR/Hx0r22i9W2Q2dcbr9o5zFbRhFg3zyUc6HAqUlt0h/F+fs3TFqCh7+0bwoDgDBtxs+Jc3fgx/VZWLrLPpsp49hZw/4IwHLH1vvtxX5psukyzn+VhdKYOHeHV7+12eamHbnsR0mHmF0AX7z9h3PGRFl7Z4ElAFQP989X1axuNbfrvWkS8uT+a67EV6ssHcfWmrJV30lLkTlhIPZmu57oLLfAdRmemrrJtm+9/63YhyNncrHn2Hm7wVw/bzyI37Ycwu5xAwBY7nQ80d8JALAL2FZpB3Nw8nw+HvrfOtSpFo7Ul/sCgF1+f/sxfwAArowu7n/RN+nsPnYOuw2adDYbTN1QVKQQFCT468R59J20FH+/rglG9U9w2m7augPYe/w8pqzYh3/ffJWnU/Wb9MNnEF+nKiJCg23LvluzH1uyTmPCHe3KrRxmeuSb9QCAzAkDTS6JeSpVDf2R65raXq97qa/H7d39sI6ZM56cNRjKXxL6Zo+yEhYchB7N67hc7zgfjqM8N7njgHHO/+jftuPjJXuxIP2oXaokYMmTj0uZhfTDZ9Akxv0FzcjHS/Zi6tr9iEuZZUtJven95bamEP3xRv+23enzP20oTsm8rqXnuzzHjCQA2Kp1SM/aethWprs/XoVch+/K2lTny4U552IBWr0yx22n7XM/bMY9n652+fn+7y7Dcz/aZyq9+MtWTF13wGn7oiLltqPblUCepiHr1AWn37IyqlQBPblNfdvrGDfNAnrbRvczXB4ZFozJ93T0S7l8EVYOD7uuEhaMrvG1Xa53NR+OVa5Bjdgfbnp/uWFe+ryne3r8rLVtHbDPzLH6MfUA4lJm4ffNzudWIyLU9nrUz1udOlrP5RXaApyrQHxJ6zx+c25x+dfsO+kURK1cxcuVe45j0OQVdrn47Uf/gdyCIlwz/k/DtNnss3n4aUMWVu45AcByl7Ng+1Fbh/bXqzIBAKu09QDsBr3FpcyyuzNq8uJsNHlxNtqP/sPlRcJI+mHXTV/+ll9YhPFz0j020fmDUgrdJy7Co1oNvyLILbhUoulBKlVAr+ah2eN/D3V2WlY1PMSwuUQBGNiugb+KZvN8v5b4ZkRXv+/XG9c2jQZgqRH/o1ezEu/nYn7pm32MXCpSaNfYPltp4h1t0bJ+dZ/24zjtAwA8r3XQGt1J6adJAIC+k5bY1VDb/Hsemrw4GwDw2m/Og9YA1x3T1jb6uz9ehRvfKe7IvlRk/B3+a/oWbD5wGodPG3d4Gg1u06f9zU07gmYvzcHDX6UifpSlzG9r/QInzxePV7AOerP6eKlzh37OxQKs3HMCj3ztXSBzHFPgC18H0v175jZ8vGQvxs8u/bQdy3ZnIy5lFuZsNR4xbe2jWrSzdJPZvTB9i+0u0nHwnC/SD59Bq1fmImn8Qp/viipVQK9XIwLTH7kGW1+70XB9w5pVDJdvNail144M82vZrGpFhqK7i+aOIAFEymaYe8t61W0pjB8t2YOgUgyn/++iDNvr+jUiSl02vRUZJ+zeJ7exXFRDg8t3+L+ruXSMcv0B4Jb/rsBj320w3ldhEdbsO4ldunTPH1KdR94CxYO7er61CHEps1weDwBqRFgqIvppIh7xUIucsnyf4XJ3U2NYB9fd/sEKt7XCsbPS8dGSPViw/SiavzQbs7YctmsGc6S/2xjuY5bM92stYz32Hbe/m3rs2w126bPn8wpxqUg5dVLnFV7C92v3Qyllu7g9+u0Gw0y1QjdTZ/hiWmpx89aZUsyq+vrvlmbDo2fy0HncAp9q6pUqoANAYlxtVNduobeN7ofpj1yD6KqW4FwrsvjWeu8bA9zuxxrw/tYjvkTlSB+TbLg8NMj1V7prbP8SHcsbt3Ro6NSZ6W2zlCP9rfXoQWXbsRcZZunEc0xvLGuu0iaNcvWtHDNmrFxlGhkF68MOg9WMxhZYWefomb/9qMttHI35fbstk0nPOtum0WA5qw37T9v6CKwcg/WEOTvw8FepKLik7C5wD/1vrVOtVH+3sWqv/YXcW/r+DKUUZm09jOFfpCL7bB5+2ZiFq/49D8O/WIc5acXlzrlYgPGzd2DUz1ttk/RZGWWq6S/uu0txF6LnWHHzZbzKiXP2I8O3urnoO6p0AV2vangIEuNq29VGq4YF48nezbyuob40sDWa1DF+mIY7VcKCcVenWKflwdpxn+jdDJ8/kIi1L/WxrQtx035uvSh5o1Et5zuRGhEheOtOSzbD1JFJAIAezVx3jOpNeTDRcPnInk1wY+t6bj/72PVN3a53p99V9WwPBS/vCbr2+3EkqDUTytHEuSVvLhjUoSEuFlwq0cNS+k5a6rRs9d6T6DJuAd6Y7T719vHvNiLr1AV8atBE486indlefacfLM5AXMosZJ3y/fvXD5LrPG6BbbqKJbuy8fh3xR39mcfP2+ZScncHBFgucJm6R1ve8I7zd+fKtkM5LmvP+uWbDpxG1zcWYvp657u2/EJLf4jV6Qv52OlwUVm2+ziW7z6ORV6MHq7UAd3KGrjCQ4KwbUwynr2xpdM2Y29t4/LzX43ogmrhIXjdx9rogLbObfAhWtPBcze2RJ+Eeqhb3dJk8cA1V9q2cQzeqS/3xfIXejvta+rIJKx9qQ/mPNUD/3vQ0j/wxUOdMf9Z507EDo2jcFdiY2ROGIikJpa29Ete3qr1bmUctK9pGu2xiejapt5dNIzo75Kf7tuixPtxx1VOvfW2Vu+vE/59Zu0yH2bvdGStST/6jXEzj5V+viBPjp3Nc5pywUj3iYswbna6T1MlAJZO2ZUZx902w1g7lbtPXGQLtpsPnMaF/ELsO34evd9ejOPn8gwHEXr7wBbrQ24AoJaHilLS+IV223tjwfajiEuZhYHvLcevmw7iXF4hVjvcgbz/Z4atvPO3WwbG/fPHzU5TOrz9x048/FWqbW6jHhMXwdFb83Zi2Odr8JAXo4cDIqCPv6MtFjzbE7XctIvf4KamGRsVibTR/RCrm9nxs/uNa60AkNCgBgCgl0EKnFFaYuaEgRg9qPiCcuK8/S1VdNUwVAkLxo7X7ZtxYqOqoG71CCQ0qIHrW9VF5oSB6NWyLiLDnDt528Y6T43gKZvFE8ca/oC29Z22KU1Lib5j8m89m+Aa7UIEAJ8/4Pr798UxF51KRtks17212C/H1Nt99Cy2ZuXg9y2+/RbWJiFPI1unrDBuM/eH027msjfy8q9pPk20dtP7y3HqfD4GTV6BzmMX4Pq3F2Pv8fOYk3YEV9T2/a7ZyMQyeBbCqzPSbK9/3nAQbf49D0M+sc8W+mJlJu77fC0+X74PkxcVN8k5TulgnVrkIa2PobTp0QER0MNDgtGsrvtMiXo1ItDvKktQH9rlCsNt9INt+hpcAKzZMtW1zioRQdtGNW37BWBrQnCnu0OgtNaCHS8G7mrH20b3s11YnujtfUZL6st9neZXccXaRNRcGwz1wb2d7Nan9G/lVEPseEUtr8tSLcL+wvTtw12RPiYZa17sgz4J7pt6AGBYkvHvaDXQ4A7KW8FBgq9HdMHqUX08b+zGDe8sxc3/XY7Hv9uItq/5/yEj1qaFstD/3WV+3Z9RrX2Mdqd0XpcFEyTAkl2WGmtEaJCtT6BuCTrozxkEyFPn8932JVidzS3Ap0v3OuXsH9J91tNdmNGdoF6+rkN2/V/G/Tq+CIiA7i3rxFo7jxjn0+priHrWuDrt79cAAG7SpTv+9kR3fHxfcW3SaNIpR64GFzkGcHfNylXDQ/BQtzgAQP82ngOXNeUzLCQIkwZ3wOwne2B4t3i8cVtbw+1HdC/uLJ7/7HWGg7Ruv7oRBic2RtWwYPz+RHdkThiIqroU0WFJV7jMXnkhuRXG3GLfDBYUJKgSFox62v+4/7mrPd4d0gHrXuqLL4d3wR/P2Dc1jb21rcvBY78/0R3vD73acJ03Ol0ZhR7NY1C/pv+yfHx5qMm/b27t075vu9r4Il2ai1p5cGyqAICXfknDHq1ylVtQhL6TlmLj/lN4ZtomvxxzxJfrkDR+od002kZG/7Yd42anY9HOY4hLmYUbJi3xS158amZx4A7XVQCN2tgdeXrWw2UV0K2s7dqOWjesgW2j+9kyZDLG9ceO15NhDUlN61ZF+phk3Jd0peHnAWCpF+2m1g7bF5Jb4aNhnZzW168RgYe7x3tMGbyrUyzWv9wXrRvWMFxfQ1cDvrerpTZrvZi0blgDr97cGvdoyx07Whu4CGRfDe9iex0eGoygIMG2Mcm22TDfvqu9bf3YW9ti97gBhkH30V5NUVOXlWTkjk6xGNShEWKqh+O6FjFoUc/4Lsw6gjg2ynIOr97UGm0a1XTqGH/ShzuZ8hjRq+c4VuKhbr5lX/2y8SD6GtzVjLvNdd8RUDx2wSyOWT+u3PXRKrs8+9LYsN/S0expf9YAO0Kb72f3sXNo99ofpT7+nR+tsk2Hve1Qcaft92uL0x5LmiNwWQX0tNH90L1ZHbw7tIPLbaqGh9gCQUhwECJCg9HvKkvbcWhQEKqEBbttCvGm+cP6Y8XXibQb/QpY2ttXv9gHL9/U2mOHpIgguprr1MTbOxZn4aT0b4VdY/sjPCTYcNvGtRnufDEAABEmSURBVO0DuquJyHq2KO43iAh1/vOp5+e8dW+k9G+FzAkDsfyF3sicMBDDdXcXC5+7zvb62Rtb4v/udv3b6/1Hd2HyxjO6Tl3rvDKe6EfIbh3dD1dGOz+dy1u9WsagSpj9b7vxlRvc9isFBwnitAyvj4aV/6hpX3h6gI0/5VwowAtuZhL1B+t02OddDLiqWsK5o7wK6CKSLCI7RSRDRFLcbHeHiCgR8U+Plp9VCw/BNw93dRnUXPm/IR2walRvr1Ih2/owb3sJRvb65J+6B2eLCMLc1Do/vLcT3h3iXbCzcjWNQeaEgU618gevjQMA7BybjIxxJc/Ht6Y3/vNG77JimsZUwxu3tcWS53sBAHp7OQWzvr32vaFXG45C1nvs+qbo1TIGU0cmoU61cCRe6fwsXEct61dHm0Y1ME1LM7XWsG8yGMEc7yG1ttMVUba0VasoLcPj1g4NDT9TpBReHJCA8be3tVVa/K00aa3+Nmmwdxfp/u8utRskZAZXzb+eeAzoIhIMYDKA/gBaAxgqIk4NfCJSHcBTAALueWLhIcFo4GIUqpW1o9WbkaAp/RPQrVm0XW23LHiaKkEvqmoYBnUobod11Sar58uo11dvam27Q3CXj+/JnVruv6tmJiP3dL0CV0ZbAmJJ5tK5pX1DXN+yLvaNtzTFVQsPsauRT3kwESHBQfjioS62lNEcDyMFrYPgfn+iB7pqn7GmvF7V0LlSEOPmTgwARvSIt5tpUe+duztgaJfGACxNSdYLe0RIMKqFh2Bolys8/pYl/Vsd0b1Jqfoy/OWbEV1dNtk5OuRlM1BZslaAfOXNX3cXABlKqb1KqXwAUwEMMtjudQATAZj/bZjgjdvaeBydahVfpyq+fTipxLdVvjAahOSNVj7Or+JJUJD7OwRvvXxTa7w8MAG9WpTsYScRocFuZ6J0R0Sw8ZUbsPrFPniqb3M8368lkq+qb5jH7y7bactrN2LDyze4XK9guXXTXzRe6N/K9vofvZpiZM8mdp8xSmXVl9vaPls9IgS7xvbHv5JbOs3N/+Mj1+COjrF49gbnu5+lu7Kx8ZUbDO9A9c1ajqqFh+Dm9g3Rop7vs2y6k6L7PjwJCwlCt2bRtqdlGZk2MskuEaC0vn3Yfj6n+c/0xKpRzmNNXLm2WR2sHtXH48A+R95ElEYA9PcfWQDsSisiHQE0VkrNEpHnXe1IREYCGAkAV1zhPuWsshGRUuVkl5W5T/fABR8nRgJg1w7tKPmq+obpYOWhWngIHu7RxPOGbnw1vIttYqvMCQMx6Y+deO/P4vlr3F3MonQDVR673nV/SYiL7J4u8bXtZn/UE9h/5qm+zfFU3+a299tG90MVrSPa03Nta1YxPsZxbVi50eRtneNqo3NcbWSfzbNNA2zVqn51RFUNw7S/J2HT/tPYd+I8XvrFko/dNKYa1r/cF53GWp7X+8pNrZHcpj7+OnHedhGPrhoOwDIvy/UtY6AAp2H5b93ZzjbJmjuzn+yBhAbVnZ4b68rW126EiLj83utWD0fXJtFY4adHDm557Ua7Y00bmYTm9ar7PHti/ZoReLx3M/zhw9QPpa4yiUgQgEkAnvO0rVLqE6VUolIqMSamfJ4+dLmrHhHqU0flon/2wq+PdXNbw/zovk745mFzZpT0B8fmhX9c38w2pwwAWzpoabgKHj9oqa9GorRmGFef1XfYt9MNJLNm9+hteMX+DiDYh7QJ/Xfx2f2J+HJ4F/z06LXauhBc26yOrUnOOhZC3znf8YpaaFSrit0o4om6h2y8f09Hw+wuxyeSGd1JrX2pD1o3rOFTc5+7+ZUAy988ANQp4dxHjqy/3997NkH18BBbk5qIuJ3W2oj+jiipiefPelNDPwigse59rLbMqjqANgAWa19yfQAzReQWpZTz872oQvPU+RYoPhrWCeFalk5EaDC2j0m25fgOTmzs7qNesQbFD+7tiHaxNbH76DnDJyHpDe8ej0itTdsTfXPd939Lsr2+tUND/LrpkFMAf+vOdnj2B+O5212VHTAeYGfZJgSbX70RkeHO7fZG9VBrc0e18BBUCw+xmx3x0V5N8UKycxOKY59A05iqdinHO15PxsL0Y3aThD1yXVPbA8J3jk3GqfMFLpMZ1r7YB9HVwm3fVRcfg22LetXsZthc9q/r7e7MRg1IwKgB9k+1mvb3azDsszVY7uZuoI7u4qi/cH08LBHP/bgJn7spkzc19HUAmotIvIiEARgCYKZ1pVIqRylVRykVp5SKA7AaAIM5VWjJberj+pbG7fD+mOLY+j/s0TO5iI2KxPWt6nqcryY0OAj3JV3pVW1aXytvrJuy4u272hs+1KWXdq5DOnu+WFnP/yoPHc81I0Pt7uRaap2Orp6G9P7Qq/H7E5bHAerP0SiYA3Dqc/mnwxxNEaHBGNiugS1LCLDUiq3HCg8JdjswrG6NCLtytKpfA50MspMyJwxEQ91+nunbAt+M6IrvdBfSyfd0ROPakR6TJwB4TE91fOIXADSpUxU1I0Px2QPus608BnSlVCGAxwHMA5AO4Ael1DYRGSMit3j6PNHlyDoAqyweogK47nQNCQ4y7GyvXTUMK1J6u52kTm/u0z3sApY3rtEGKbnKfb+5fUNb3rs3nruhBTo0Lp5KwlUqetcm0Ugb3Q+rR/VBVNUwZE4YiJvbG6dqPtmnueFyq2kjk+yas6z+1JplACDtUA66N69jV5OOrub9bKm1XUwY5qqpb/kL1+PXx7t5tW+v0iyUUrMBzHZY9qqLbXt5dWSiCmbS4PY+pXq6kxhXu8wfVrzon718eqKNLxlPrep7nxZq9eKABNzcvqHHB6Fbjb+9rW1eJKs372iHf/1k6RhtElMNvz7WzdYUVreG6zZua1OOJ8/e0ALDu8W5TPEMCQ7CzMe7I37ULLtxIvrt9R3OT/dtjgMnL9rSVb1xhXZH9cZtbdHvqnq2/oejZ3LxvxWZ+O899mmesVHeDzgr+7w5okpCP7K2MoivU7VC9XmEhQQZNlm4YtRXMLhzY1tAt+rZIgZLd2Wjc5xvbdyuuBs9a3VdixinLBwr/Xdekmmf7+wUi3o1ItCjeR275r16NSJKXQlgQCeiCmXti32Qp5veWD9/UHkxCuZ3JzbGtNQDthp2SYlImQ0qZEAnogqlJNPkloeJd7bD/ddeidYNfG+OKi8M6ERELkQ5zAhqNC1DRXJZzbZIROQNawdy13hzpxf2FQM6EZGDx7VpsDv48ASuioBNLkREDgYnNoZSwF2JlSvziQGdiMhBcJDYnuZVmbDJhYgoQDCgExEFCAZ0IqIAwYBORBQgGNCJiAIEAzoRUYBgQCciChAM6EREAYIBnYgoQDCgExEFCAZ0IqIAwYBORBQgGNCJiAIEAzoRUYBgQCciChAM6EREAYIBnYgoQDCgExEFCAZ0IqIAwYBORBQgGNCJiAIEAzoRUYDwKqCLSLKI7BSRDBFJMVj/rIhsF5EtIrJQRK70f1GJiMgdjwFdRIIBTAbQH0BrAENFpLXDZhsBJCql2gGYDuBNfxeUiIjc86aG3gVAhlJqr1IqH8BUAIP0GyilFimlLmhvVwOI9W8xiYjIE28CeiMAB3Tvs7RlrowAMMdohYiMFJFUEUnNzs72vpREROSRXztFRWQYgEQAbxmtV0p9opRKVEolxsTE+PPQRESXvRAvtjkIoLHufay2zI6I9AXwEoDrlFJ5/ikeERF5y5sa+joAzUUkXkTCAAwBMFO/gYhcDeBjALcopY75v5hEROSJx4CulCoE8DiAeQDSAfyglNomImNE5BZts7cAVAPwo4hsEpGZLnZHRERlxJsmFyilZgOY7bDsVd3rvn4uFxER+YgjRYmIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDBgE5EFCAY0ImIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDBgE5EFCAY0ImIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDBgE5EFCAY0ImIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDhVUAXkWQR2SkiGSKSYrA+XESmaevXiEicvwtKRETueQzoIhIMYDKA/gBaAxgqIq0dNhsB4JRSqhmAdwBM9HdBiYjIPW9q6F0AZCil9iql8gFMBTDIYZtBAL7UXk8H0EdExH/FJCIiT0K82KYRgAO691kAurraRilVKCI5AKIBHNdvJCIjAYzU3uaJSFpJCh0A6sDhu7lMXK7nDfDcL8dzL6vzvtLVCm8Cut8opT4B8AkAiEiqUiqxPI9fUVyu5365njfAc78cz92M8/amyeUggMa697HaMsNtRCQEQE0AJ/xRQCIi8o43AX0dgOYiEi8iYQCGAJjpsM1MAA9or+8E8KdSSvmvmERE5InHJhetTfxxAPMABAOYopTaJiJjAKQqpWYC+BzA1yKSAeAkLEHfk09KUe7K7nI998v1vAGe++Wo3M9bWJEmIgoMHClKRBQgGNCJiAKEKQHd01QClZGIZIrIVhHZJCKp2rLaIjJfRHZr/0Zpy0VE3tPOf4uIdNTt5wFt+90i8oCr45lJRKaIyDH9OAJ/nquIdNK+ywztsxVikJqL835NRA5qv/smERmgWzdKO4edItJPt9zw719LPFijLZ+mJSFUCCLSWEQWich2EdkmIk9pywP6d3dz3hXzd1dKlet/sHSs7gHQBEAYgM0AWpd3OcrgvDIB1HFY9iaAFO11CoCJ2usBAOYAEABJANZoy2sD2Kv9G6W9jjL73AzOtSeAjgDSyuJcAazVthXts/3NPmc35/0agH8abNta+9sOBxCv/c0Hu/v7B/ADgCHa648APGr2OevOpwGAjtrr6gB2aecY0L+7m/OukL+7GTV0b6YSCBT6KRG+BHCrbvlXymI1gFoi0gBAPwDzlVInlVKnAMwHkFzehfZEKbUUlmwmPb+cq7auhlJqtbL8hX+l25epXJy3K4MATFVK5Sml9gHIgOVv3/DvX6uN9oZl6gzA/js0nVLqsFJqg/b6LIB0WEaIB/Tv7ua8XTH1dzcjoBtNJeDuC6osFIA/RGS9WKY4AIB6SqnD2usjAOppr119B5X5u/HXuTbSXjsur8ge15oVplibHOD7eUcDOK2UKnRYXuGIZTbVqwGswWX0uzucN1ABf3d2ivpPd6VUR1hmpXxMRHrqV2q1jssiR/RyOlcAHwJoCqADgMMA/mNuccqWiFQD8BOAp5VSZ/TrAvl3NzjvCvm7mxHQvZlKoNJRSh3U/j0G4BdYbrGOareS0P49pm3u6juozN+Nv871oPbacXmFpJQ6qpS6pJQqAvApLL874Pt5n4ClWSLEYXmFISKhsAS1b5VSP2uLA/53Nzrvivq7mxHQvZlKoFIRkaoiUt36GsCNANJgPyXCAwBmaK9nArhfywRIApCj3bbOA3CjiERpt3A3assqA7+cq7bujIgkae2L9+v2VeFYg5nmNlh+d8By3kPE8vCXeADNYen0M/z712q3i2CZOgOw/w5Np/0WnwNIV0pN0q0K6N/d1XlX2N+9PHuMdT3BA2DpLd4D4CUzyuDn82kCS6/1ZgDbrOcES/vYQgC7ASwAUFtbLrA8NGQPgK0AEnX7Gg5LR0oGgIfMPjcX5/s9LLeZBbC0+Y3w57kCSNT+B9kD4L/QRjSb/Z+L8/5aO68tsPzP3EC3/UvaOeyELmPD1d+/9ne0Vvs+fgQQbvY568rWHZbmlC0ANmn/DQj0393NeVfI351D/4mIAgQ7RYmIAgQDOhFRgGBAJyIKEAzoREQBggGdiChAMKATEQUIBnQiogDx/6fX/5WwYanIAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGn6QgmEEO9t",
        "outputId": "6ea2e45d-7c0e-49d9-9a00-539be9da8172",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 325
        }
      },
      "source": [
        "learner.save('first_cycle')\n",
        "learner.freeze_to(-3)\n",
        "learner.fit_one_cycle(1,max_lr=2e-05,moms=(0.8,0.7))\n"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>error_rate</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.413655</td>\n",
              "      <td>0.402917</td>\n",
              "      <td>0.850044</td>\n",
              "      <td>0.149956</td>\n",
              "      <td>1:17:41</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd3wUZf7HP99NpYReDSWhSO+hCEgXKYoVxY7l1Ds59dDTWA4beqin3nGHBRTL/URULHDSBAEBQSBIlxYglNBCCTV19/n9sTOb2dlnym422c3wfb9evNjMPDv7zO7MZ57n2x4SQoBhGIap+Lgi3QGGYRgmPLCgMwzDOAQWdIZhGIfAgs4wDOMQWNAZhmEcQmykPjgxqYZo36pFpD6eKQVbss8gMS4GLetVjXRXGOaSY/369SeEEHVl+yIm6PE16iMjIyNSH8+ESEGxG62eXwAAyJg0MsK9YZhLDyLab7SPTS5MUBS7OW+BYaIVFnQmKFjQGSZ6YUFngqLI44l0FxiGMSBiNnSmYsIjdCaSFBUV4dChQ8jPz490V8qcxMRENGrUCHFxcbbfw4LOBMX5gqJId4G5hDl06BCSkpKQkpICIop0d8oMIQROnjyJQ4cOITU11fb72OTCBEXOucJId4G5hMnPz0ft2rUdLeYAQESoXbt20DMRFnQmKOJjnX0jMdGP08VcJZTzZEFngsLDJnSGiVpY0Jmg8LCiM5cwubm5ePfdd4N+34gRI5Cbm1sGPfLHlqAT0TAi2klEmUSULtk/lohyiGij8u+B8HeViQZYz5lLGSNBLy4uNn3fvHnzUKNGjbLqlg/LKBciigEwBcBVAA4BWEdEc4QQv+uafimEGFcGfWSiCA+vcMVcwqSnp2PPnj3o3Lkz4uLikJiYiJo1a2LHjh3YtWsXrr/+ehw8eBD5+fl47LHH8OCDDwIAUlJSkJGRgfPnz2P48OHo27cvVq1aheTkZMyePRuVKlUKS//shC32AJAphNgLAEQ0E8B1APSCHhQsCxUTFnQmWnjpf9vw++GzYT1m28uq4YVr2xnunzRpErZu3YqNGzdi2bJlGDlyJLZu3eoLLZw+fTpq1aqFvLw8dO/eHTfddBNq167td4zdu3fjiy++wLRp03DLLbfgm2++wZ133hmW/tsxuSQDOKj5+5CyTc9NRLSZiGYRUWPZgYjoQSLKIKIMd7E7hO4ykaYsTC4HT13Eyt0nwn9ghiljevTo4RcnPnnyZHTq1Am9evXCwYMHsXv37oD3pKamonPnzgCAbt26ISsrK2z9CVdi0f8AfCGEKCCihwB8CmCQvpEQYiqAqQCQ1KgVD/UqIGUxQu//5lJ4BJDF1RuZIDAbSZcXVapU8b1etmwZFi9ejNWrV6Ny5coYMGCANI48ISHB9zomJgZ5eXlh64+dEXo2AO2Iu5GyzYcQ4qQQokD580MA3cLTPSbaKIsoF3a0MhWFpKQknDt3TrrvzJkzqFmzJipXrowdO3bg119/Lefe2RuhrwPQkohS4RXyMQBu1zYgooZCiCPKn6MAbA9rL5mogcWXuZSpXbs2+vTpg/bt26NSpUqoX7++b9+wYcPw/vvvo02bNmjVqhV69epV7v2zFHQhRDERjQOwEEAMgOlCiG1E9DKADCHEHACPEtEoAMUATgEYW4Z9ZiKI1uQihLhksvYYRmXGjBnS7QkJCZg/f750n2onr1OnDrZu3erb/uSTT4a1b7Zs6EKIeQDm6bZN0Lx+BsAzYe0ZE5UIjaBvO3wW7ZOrR7A3DMNo4UzRCsT6/aew5dCZiPbBrSmHfvJC5At1bThwGinpc/HxL/si3RWGiTgs6BWIm95bjWv/szKifdCaXH7afiyCPfFyw7urAAAv/a9UaREM4wgiJujsWyt/JszeilWZpYv31gr6Z6sN16oNiaNnnL9oAcOUJTxCryDc98m6Uh/js9X7cfuHa0p1jLJMFF2+O6fsDs4wlwAs6FFCYbHHz+Go5UJBMZbsOF7OPZJTlqn/T83abLttQbEbFwvNCyIxzKUGC3oUkHXiAi5/fj7eXbZHur+wuPQLM8seFlOWZuI9g880YlsQtTM2HDiN06VwnL6zaBeG/XO5dN/Qd5aj7YSFIR+bKR37T17Agq1HrBte4lStWhUAcPjwYdx8883SNgMGDEBGRkZYPo8FPQp4dZ43D+vNhTvL7DPckoygNxfuxOsLdgR1nCoJ9qtF3PDuKnR5ZVFQx9fyr592Y8dReVbe/pMXQz4uY82mg7mGM0YAGPbPFXj4/34rxx5VbC677DLMmjWrzD+HBT0KiDFJzhFCWIqiEAJTl+/BKZPRcJE7PKaSulXjbbXbk3PecN/bi3bhwxV7pftS0udi86GyXwiAMebnXTm4bsov+O+vxk7vvKJLs7heeno6pkyZ4vv7xRdfxMSJEzF48GB07doVHTp0wOzZswPel5WVhfbt2wMA8vLyMGbMGLRp0wY33HBDWGu5hKs4F1MKeqTWwoJtR6X7vso4GLBNn6G58WAuXpu3A7/uPYXpY7tLj1Potm+2+fdPu5FcsxJu7NooYJ9spH++oBgJsS7ExZSMD06cKwhopzL5J28FugeubCYdBY76zy9Y8dRA2/1lwsvWbG+uw4YDubj7CvO2Ec0Wnp8OHN1iq6mAgBCAy6qvDToAwycZ7r711lvx+OOP45FHHgEAfPXVV1i4cCEeffRRVKtWDSdOnECvXr0watQow+/lvffeQ+XKlbF9+3Zs3rwZXbt2tXUOdmBBjwIaVE8EALSsVzVg38FTgU9vjwBiNNdKsSKyZ/OKDD+jWCPo90xfi593GUeUvLVoFwBIBb1YJ+hvLNiBd5ftQYt6VbF4fH/f9vMFgQ7L7zdkIzGuRPQLit2IdckniQP+scywf0zZoj5wv9uQjXdu7WzatsgtKsTC4flFHrg9AlWDMBnK6NKlC44fP47Dhw8jJycHNWvWRIMGDfCXv/wFy5cvh8vlQnZ2No4dO4YGDRpIj7F8+XI8+uijAICOHTuiY8eOtj7b7REo9pgPzFjQo4AYl/eGaFq7ikVLLx4hEIOSm6hYMaccMYnj1gqxmZhrySt0o1J8jN82/QhddeRmHj+PwmIP4mO9Ar3o98Cko8e/3Oj391+/3ox/jO4k/WzZTMCMq9uVFEnyeARaT1iA9GGtcV/fVJN3RY6fd+Xgya83YflfBwZ8xwCQX+RG678twPirLsejg1uaHuvZ77Zg59Fz+OaPvYPux/cbslG9UhwGtq7n21YQhBM+ogueDJ+EYrdXqBPiAr9DLbsVM17HRqVfBm706NGYNWsWjh49iltvvRWff/45cnJysH79esTFxSElJUVaNre0jP14LVZYrBsQORs6Zxb5KJFme1+KXuxUgc7OLRnNn7pQiCNnSv4uKAo+UkZ7PJVlO40fBsfPlVzEM9cFmor0LN+dY0sQgi3ZuyfH+3B5+Qfj7NHTFwpNnX6lYenO4+j/5lIUmCzi8urc35FzrgD7T10I2JeSPhdPfr0JAPDJqizLz5ux5gDW7z8dsN3jEfjg5z2YseaA4Xsf/3Ij7i1FjkOwD95w8/uRs9h57BwKbNr03RYjXDvceuutmDlzJmbNmoXRo0fjzJkzqFevHuLi4rB06VLs32+ecNevXz9fga+tW7di82Z74bpWYg6wUzQqUG1tWn1Zu+8UUtLnYuexwCiPNxbs9BMjmSh2fWURrvj7Et/f/d5cGnS/hrz9M8Z+vBZAiclm9d6TAIDbekgXpcK05XuRkj7X1vFjiAJMODJOnDe2x6uQ5rFoZiYtKHZjT855dHllEaYulztmS8sLs7dh/8mLaPX8AizdeVwq7Opp6226J5Vz/WGzcUjgvC1HkJI+F7kXzUNCJy/Zjb/P34Fnv5PbmX9VfsvS4I6SJQnzbc4qgnAlGdKuXTucO3cOycnJaNiwIe644w5kZGSgQ4cO+Oyzz9CqdWs/E6eKEN5wz4cefhjnz59HmzZtMGHCBHTrFr7lIyKY+h8dF0K4OH2hECnpc/FJCEWi1IgBbeTAvC3eG1qWqj/9l33YfqRE6LNOBI7ywsWynTlISZ+LFs/Nh9sj0DO1FgCgcnygte7k+UJfCKaWpw0ShojI1ghvbdYpyzba62lvjvH30fmlRRj81s8AgJU2yiB4PAIjJ6/At78dsmyrEusqEel7P16HF+dsC2iTedwbBeTSPXzW6c71jMQvMk2JENJHEunzFVZlmgt2OLKPzWZPJ84XYOnO6EiIUwnXrGzLli1YutQ7SKpTpw5Wr16NLVu24OOPP8ZXi1bjYoL3Pjl/3vsbpaSk4Lslq3EmrwgUG4+ZM2di+/bt+Pbbb7FmzRqkpaWFpV88Qg8T7//stSW/GEKRqFcU08CqPSU34Ko95mLj9gikpM/F07M2B4xINxwInH5bUVjssTRt5BW50alxDSTGuXx2fy1ZJ+VC+mXGQemIxUXAXpPwRhV9xMyCrYERQQu3HfPdrHWSEgL2q2gfmntzLuCRGb+ZfteFbg+2HT6L8V9tsuynSmyM/3ezNdssGcu/baOalf3+lj3w1Hfotelcvr/46/uh52KhPTPF4LeW+a5vPWYzrDs/XIN7P16HIuW3v1BQHPDAkmGWNW2IECgsdlvOWiJpIcpXrr2zecXIvVg2Jj8W9DBx7GygE2TVnhO2YqpzdIKVefw8dh3zCt0Fg5vurunemixfZhwMmLafzQ8uJb7Y7cHlz883tTkDXnHxeARiiKThX41rVZa8y4ssbtlFhAf/u96yf5OXZPr9/fD/yd/T8aUfkZ2bh7pVjQVdS3ZuHuZuPoJ7PzYeqYZyz+lnCGazUf1vbycC0CgcTv+QPXAq+OQr2QNkT84FTJovT0DzeATOXCzCUklpit3KLEQ95uNfbsTo91f7zEoy8ovcuPz5+Uh9Zh7+/VPgAstm7Dh6DgdOXTQdmJSV3yQYjp/Lx4FTFwOi0oQQuFBQXKo+VghBzy+K/rod3288HLDt9mlrMOo/vwR9LP1IS0buRW8bFwEtdOGOwV4Qp5Vjfb7G3Jnj9gi4hYCLCDGSK+fMReN+y3oU47JncnlmeGsAwNn8Inyz3tj0cS6/GEPe+hmbNTXj84vclt9nQbEHW7PPSK8xrX+iw4sLfc7HIrcHn67Kks489KNWs5+jVhV7iVobD+b6+qfK9nKdk4x0o/2EWPu3t+r3aP7sPIuW/riFwF3T1+DeT9b5RqC+fcr3oP6v1vI3y4m4oAl3/cjAfCm7voXBaz2nLEbw5Umxch75RW5sPpSLw2fysSfnPHIVoQ9F2CMm6HacYSo9X/spaup2pKTPRUr63AD75fMj24Tl+HmFbuvkBw1EhH/rRrDab3bbYesFMdQb0UWEHzYHPphU1BG6y0U4L5kF5OYZ3yz5BjMNO4Kutrnp3VV44mtz00dekRuPzChJSb/jwzXo8OKPlp9xzb9XSq8xrdPvXH4xbnpvFZbuPI6PVu7DC3O24Qsb0TzFJlm6eoe27Ps4eb4A10/5BV1e9mYMq6aSyboR7O7j/g70YBN+rAQk83igg97tEb4HaK7BA139Do8qs1izzGirPicmJuLkyZOmfTWLLjLLprbibF6Rz3wUDtQHsOr0V2cuRW6vyenkyZNITEwM6pgVIg5d5hiKNG/9uBPv3lHinVZtnx1KuSTb32ZvxdjeKbbb6y/sgmK3n6KPnLwSK582z7pUk4BiXIRxMzYYtnN7hDepyUX4VFILPa/Q+GIfMzVwBXRZWKSM9G+3YEyPJr4pfDDIwvmC4e0fdwVsu/fjdXioXzMAkD7Y9KiRSmOmrkbl+Fi/bN7PVmfh7zeWJJbIBjrdJi4GUBIfflRj3tM6Qv82exu2HzmLHa8MQ6IuLju/yB2wTY/sYaK9vg6ezkOLekl++/u+XhI9tS7rFK7tdBkA/1mmW/dAMxNtq0dQo0aNcOjQIeTkeMNnj532XkNFJ+N9Yl18Kh6VdOeqtgOA7ecqWXxKIEIIZOfmI8ZFaFg9UGSF8JrWXES+z9J/jrYPAFB4Ig7HEmKRe7HILxGvoFIcTibGIjExEY0aBSb3mVEhBD0aOHWhEF01NVUSYksuGI9H+Oy6W7JLt0Rc1okLtuyoRqzYdQL65EuzESJQIuhWTrLs3Ium9T20WaB69pZhJE5ZYhQHrgrvJ6v24Za0Rqhtw27/616vQ1Ab1vnF2oN4uH9zX1KZ1W+l5+DpEjv59iNe5+vkn3bjqWGt/aJtityeAEHv1rSm3wPvnOTh9M/FJbOABJmdTYPWd9DtlcW+1/qH1PB/LcfypwbiD59l4NSFIsx/7ErfPm1L2Yg/Li4OqaklyWLDle/y7Vs6YfycktnbL+mDkFyjUkA7AMiaNNL0PGQcOZOHEZ8tMXz/hNlb8dnq/bijZxN8vuaItN1wXTjvmzd3xOjOjfHnLzbgf5tKwlRHdGiAd+/oEHQfgQpiQw+GsR+vNSz8ZMXuY+dw1sDeqo/G+G5Dtm+qtNCgDkuoGKXDy9APqr5YeyDAZnvYYiT8hs2Kize9t9p0f3wQNttIsN8gCicU1NHssbMFvhG0SryF8Okpcnuwdt8pnLlYZJnavelgrp/ZQLbGrBpJoq1UqR72cG6eLwpKP3uRFYH7l8asYxU++s7iktmM1k6uNyudOF+Iacv34ZfMk76HkFFblR82H8ZL/wsM/1TRh9F+vyHbtK/Bos3pkKGu3vXFWuMkLj3qTOV/m/zNnMt3+ftGpi7fYzu3I7rvQB3aTESVD1fs9XPmLduZg4lzA2OhjTiTV4TUZ+Zi2c7juOqd5bjlfa9onbpQ6DdtlI2a1SJG+SY2OxlPfr0Jf/pcHqmRsf80Dp0OvTTsbwdOB0TGJEpSy7WsyyqdWULlSxv25FAJR034/m8us9XuTF6R77c1Qm+eeOWH33321WAKoQFeU8otH6zGvZ+stfQpXDfF38muL6cAAL8dCIysUh8UV76x1LcOa7Coo/VgnXUyM5LRg8tI0MfN2ICPf8kK2K7WZpGVTygr9AKsJZiwSKOJuL4Okt5HZkZUCvrgt5ZJV3G/68O1Adsmzt2O577banq8wmKPoTMjI+sUhADGKqFr6qhGn2kp+/pdkgxPwGuv7PFqyajtmW83+90Es9YfwrwtxqP6+z+VF7u3MwIucgu8rIuFr1xOF7tZarKZOcYOHxjEQZcFd3+0Btf8e6VfxIWer9f7P7w+WrkPczYelka9WKGWNv79yNmgTS4yZA8F1TEZjlT9TAtfhl7w9TZ0PVonZrCBHeo9oZ+F/p+JabC0/PkLYz9TMBB5gyD06H1owQRJRFTQNx/KlcZp78m5IF3FXZYGb4fLn5/vyw7UY7SAAiCvGKhF/Z71N0l2bh6Oa+KLv1h7EBsPGsejx1kkgKi0v6yaZZvCYo+tVPnyJj+EWjJa9oXRXGLFJsWMkaYzpWiRnc8TX2/CUIMVlsyiI9Trx2WzFIId9PeVvljaTpPr3ozci4V4e1Ggo1iLfoYoKw+gHXW2en6BbzYc7ANHdRPo80DMCtUFw+yN2QG5AsHw4Yq9WGaSLdtmwoKAbZ+syvKV3AACs4nNiKigj/rPLwFx2vpYVrsUFLuRYWLjM0qysDsiki23pT459RehbEoqm0qevlCI3IuFthafmDOuj6/MrhmX1QhsE0phLism3Ria06ZPi9qoX81e4o+Wb38Lr03UDqEs4mBUdsBMqPwEPUxhcfpZq/7vqw0ePFZ0fnkR5ksydbVM1c2m7BTEUkNL9feJfgQb+GD03oNa561ddhw9i5T0ufg646BUd7Jz8/DYzI3o/qrxg92Mm99bhYlzt/tm/8GgFsEb+s7PvjwRO0SdySXUcpy3T1sjtSeaJbsA8pBIvSAXuz2YtiLQBFTsEXh9wQ6kf+tf/CjnXGCsa2Fx4Hl1eWUROr9sb4m2hNgYPD7kcst2WZKl2eykWwfLlZfXDel9b9zcyS9C6FLhVRO/jmpPJkJQN68ZpY22CpUitwc5uhnisp05tnwgbo8IMLm0mbDAL/NTDRNdsuMYtmafMTWLaakjiUIa9s8VAIC/ztoszRa2W8HRiIxShsyeLyj2ZYzbJerCFkO18RnFG6/INK/9PV1iq9dfVK/Nk0eB3DM90KYPALdNC4y5tmtWMUJAoFV9a5OLDCvTUbD0bl47IM7XLi6Sp8KP7Z1iq1RsqNh15Nm1vTarUyWoUEyzcM/bp3nLOBBgWBmxotDyufkB2ybO3W4rUGFd1ilpJNLIf6/0vVYHfPd9Yr2ocubx8xjy9s9Y9Jd+SK6R6DNFZp244KsaqqL/GzD3Wb27LBNvLPCuAbz71eGWfZHlXFjVB0r/xl5ZXS2OEXQ967JOYfT7q/HXq1sF/V79LEEm+sGy/+RF5Bd50LdlnZDeX5oSFLLFJupXS8Cxs6HZBokC64Zoua9PKrYfOSu9SYwcPFe3a1Cmgm53TdXnvzd3sKucC/NDEggus7NKfIxhnR8jtPXxoxFZ8hkAv9BGj4BfAa5qibGGtYuGvO31m42bscHP/2a1Gtb2I2cx9uO1GNS6vmEbVcwBebE4PXd+uMayjZ49JlVDjYg6k0u4BH32Rq/NVV/z+b+rsyx/gHCPaAGvw+zOj9ZgyY5AcbVDaQR9n2Qk2bxu4HJ3dilyC1SvFGe4v2pCDMb2SZHucxFJz0Uty3t/31Tc2yclJDu7GVbx3cFSGkeZEcFkRIfiPLWKpa4ICAicOF8i6Mk1jQvCqQQbTPHBz3tw7GyB7Zjy+tWsfVtmBcmMyAuhflVUCbrHIwJGk7Kp8rX/XomvMw4G2IaHtCl5oqpv04Yzzd18BH+bvc2X1Xn1O3LHkLaMbbCr5VhhZ6ooI6WO9YVrhGwdxX+N6WJaf+aft3ZGx0byMgZr95nb5F0uwtXt5OspGg3sXS7CntdG4PmRbfDCte3w0qh2pp8RLNmno3t0GixGv43T0T+X40tpypThCiasBMBHK60TGe3OELXI/GFWRJWgv/fzngAHo15PP1q5D1uyz+CvszZj9PurdW1LGn+uLLulnbZoizYBxk9u7XGOSsriRgI1E27B41dK98ucPioDWtVV2pRU9qublIAHrmxm+J7qleJwr8Eo2wr165PZ2V0GpXcBrxlHNTtc3a4BXr8ptEgaGT9JyruWNe/fGb6VaPSEKxmsIqC9br/fmO03i48LMivXjC/XHUDuxcKg4r4Bby1+K0KJmAqFqBJ02eLF+tKnr5jU7C7tit4q2tRqfWpypGndQO4YNRup7Fee9O0u847qrunY0Lfvxi7J8jeRdQmCf40xXxFedl+4XGQrrpaIcGv3JtYNAXxwl7VwGtXzLkuGtZfPUsLBA2W8+HW47qVwcF3nkmv098Nn/coCBzuaNuPpb7Zg/FebMMukRHO0Y0vQiWgYEe0kokwiSjdpdxMRCSIKaT0lWYjh6xrngxVzTFJyg2HC7JKaEUZZm+Hm2RGtMaa7fJ3O0rJGMZGoySTaWN5mdatI39OoRiW/4k4yruucjHaSZKcEJStUNtJxEfDE0OAd1TIGKjMPWR+chl7A2zQM/pxHdLD/gCkLP1KoaGPzC4s9vusLALYfDu+Aqyx8I+WJpaATUQyAKQCGA2gL4DYiaitplwTgMQDBu3MVZHWMtY6JmpWNHXHBEg0rl2hpn1wdk27qaN3QgOSa1iVB2yrC1z2llm/bvX0CR3pdmtRAy/pJiLUxnZWJ9n3KMWWPAxeRr8RqafngrjRseXEoGtWsjJTalaNS2Lun1AzLcaom+o+Yq4QwgjYrN6Hlk3u7Bzi9kySfZ/XADxcFmhj2xduP+Zlhwx1tFKn4/XBhZ4TeA0CmEGKvEKIQwEwA10navQLgdQBBG52zc/OQX+S2dAIEa9syI9QyAmWFrKaDEbLaLLJFm/Wo3582PK5KQiwe7t8c4wa28G1LUUq52rlhZXZytUyr1OSibAw1jl1LfKwLSYle4Vn214GY+6jcv6CnR2otv9KmU+/qhtYNkkze4S2FGkrZ1XDZuntoHsLVK8XhqrbGIXXB8PXDV6Cubg3WAa3qBeRNxEhMevMes/d9l5YfNYESxR5huW7opYwdQU8GoK1EdEjZ5oOIugJoLIQwrfFIRA8SUQYR+dkx+kxagj9K1ok8qEvX149SSoNVpEZ5o9oCHx/S0rJt58Y1ArZd0by23983dg20jd/TuymqV4rDcJ1tN11Z4k1FdQrbiQv/122d8WA/uXP1ypaB2aSqWV511IbKxOvbh/xefd3vFvWqGpqeSkuwpXT1/JI+CAsf7+f3+256YahpHoCWt0Z3Mt1/eb0krHtuSMB2vbNRVpu8frVE1DNZkDtc6FcZGjl5pUHLisnD/ZuH7VildooSkQvA2wCesGorhJgqhEgTQgTY2JfuDHSI6pfY2h9CGI8RWjt5NJCppPj2aRFa4tFDOlGtUSlwrcpW9ZOw6YWhuKxGoHmmiWaBZ9UaZWTG0WpJw+qV8PSw1tJ2b93SCYvH9/cbjZd2ljXl9q54/85uuLNXU+n+n/86wPIYqqM7SRkgpNapghOScg0y9Ot0dmlSAz/8ua9h+65NAx++wZBcoxJaNUgKejk5lfUHzGcI1Q3MmHbWI3UR8O4dXUPqV1mw+cWhke5CSGiDFKz4u0UNJTuCng1A661rpGxTSQLQHsAyIsoC0AvAnFAdo1qWSaJenIpas9yO4Mma6G942cDQbFTXLrnE/vyk4rQ0Sj4K/Cz5cRPjYtCiXlW/B0Cwgq6fsYzs2NA0ekRd+ccOm18Yij2vjQAR2V4lKkk3S/zuT32QUsf4M2U+Ciu6NQ2P3R0AjplUHdSGA+qpaWPxaiIK2oZ9TceGfqYru4tk26FaYvh8bOWJbEk7I1pZmAbtCPo6AC2JKJWI4gGMATBH3SmEOCOEqCOESBFCpAD4FcAoIUSpw0Mqusc5GNRMyV0S2/5I3RP8kYEt/Kby8yW2TFk4l1kYorpgbesGSWhS2zta14uXStNawSU5aR8AarfsCqidgmShQkS+h9FpnV22t8bEofVZyML5zHwNMvOYFWZT8KvbBWc7N3vY3KqJqlKjaFSHdv2kEpGpZnAdEKyXpdPzw2b/qqUTrgmIrwAAfHRPWlB+gp0Th3XBZMkAABkxSURBVAXVDwDoG+JsONzYWb5QxcqvZflrCCGKAYwDsBDAdgBfCSG2EdHLRDTKdk8YU1rW846GZdUmz+pSwns3r4Ndrw5Hc8XuKxv1tpWEtZmFlddJ8o6UBrSq59smu3Su7XQZZj7Yy/hAFqjiPv6q8IQummF0ww6U2O9v7ua/GK/WLLXw8X6+17KqgTI7edcmXiEPZSUdt0fggb6pAb6JzFeH4707AmPutYL76X09/PY9Oqil4aLjNSuXjI7V9HWZXiSZjHxVk43Rw98K2YO9akIsBrepj2l325/kh1LBkwhoZCM6bNrdafjnrZ2R+epw6X2l8vJ19rKbZWUz1PvfCqvgPFuPVyHEPCHE5UKI5kKIV5VtE4QQcyRtB4RjdF5etLD5RaoseaI/po8ttTUpAFXoZD+Y1UxFdhO2blANGydc5bfNbIReLykRa54d7FfMTPaguK1HY9SzUbvCihb1quL3l68u9XFkNKpZCQ/1b4b3DRKOqkoESv/dpGpGto01MxKZX8HlooAIGHVknGhTaK7UFG0r9njw/DVt8ewI/9IMsTEu6cxr0fj+vtet6vtPyatXjjMcBZuJg9b2bzRSzi9yo91l1fGP0Z3w/SN9jA8m4auHrsBD/ZphaNsGGNauAT65t3tQ79eSFqKJiojQIdm6hMJVbevj+i7JiI1x+c1q9Nx9RQru7GWeDNemYTX89ETJ71Ul3jgiTMZ+g3UdVKIqU1RlSJt61o3CxONDWgY1jW1cq7JhFbZQ62toa9AMlpx7f4Pa48Pbe00x6pRNK0Juj0CNyv72Sau1p+tXS/Szh6sX2aDWJX1qb+MG0NPYwERjJ9QyFFY+PQjPDG+DqgmxAULbqn4SHh3UIuA92mSV+/um4uH+zbH1pauxRHPzAd4Hnx3UKJz4WBfeMMgvaKb5vQZqZkbBLkNXv1oi3ry5I96/s6t0ERSXi4IOc3ygb8nswCi+Xx3h39ytke0Rumoz75FaC8+MaINK8TF4/65ufjPDYNdEqK3xBXz3p96YfFsX2+8N1u4ea1E7ZuL1HfCxycOpTYMkvzIdozp7czLU71udlfUzuOet6spHpaA/2C98YTxWCOEvWFaoNiyZQ2nOuL6+qbbKKzamYfU0lQUbaEa/amjfIxIBAoDxV12OTROG+m4SbWSC7KaIs1J0Haqga29Woxtg8XivWaJXs1oB+z67rwf+OKA5nh0hj4YpTxb+pR9a1g90LKlRQYNa18PfrmmLGBehakIsmukcw1USrEfcTWtX9ntY3WIwqtPWyK6mmYanhZCMNDqtMYa1N46W6GQx2BikDCTUNHuXi3wmK/2VpGY0a2cLtSrbc26+dYt5GCUQfCVJrUmoS5OaGGUzca1htUQ8f41xgToZnRpZ+0S0D2fA38RSpDu3L9Z6I8Jv6d4YWZNG+sJAjR4bpbahR4LyrCPhEcKXCGMH1TTyS/ogv+2q80s/Kr5JZ5uV9kHzI2sdiFNu74of/tzXUERdLvILO9OaSGSDnGDrXvRI9ToGx9ioqdKiXhJ2TRyOzx8ItK/XTUrA08Nah/ygDiZlXcaPf+ln2aZ1Q6/IX29U20bBrKCZKiR2assA/hmQ2izoRjZKwgLAx2O72zZVxEge5tra/M3rVkXWpJHooBH+JxXz25W6Gv6v3dABuyb6L+pgJ6sYCBQ7GXZWN9ISqv3+iaGXm/oHAOAuXXhs++Tq2PGKtQNW26e5j5aEtY7t7X88vcl3TI8mGNXpMjw6WD6Is5ppRaWgt9VN8W7vaa9IU6iEEomgd8KojkK1pMCHd6dh58RhtswK0tF0DKFKQmxQJg7tdFC2MG+wJNeohKxJIwOSloyIj3XZTngJhnfv6IbezWvjo3tC811cLhmR62letyp2vzrccnSnTbnf9pK/D+DvN3bAe3d0NSygBgBv3FxiftGa0kKpnTKwdT0/U4Vvu8Tpqx3ZbZxwFba9dLVlPZjOjWsga9JINKzu7zdwuch0NZ/yxkqUjVAHT/ocDi2DJCZQOwPAKsp9/1C/ZkhKKOmfvnb67T38ta1qQiwm39YFtavII1+s7q/o+VVMSIyNCapOx+pnBuG/9/ewbgivmNpJQ+/TorZh6vf/xvX1/ciqjLpcJaLfuJb3hjBKwtA7Zmb8oWfADMAOr91QknSgTvO020rDsHZe51VZYEccZvyhFwa3CU+6uxF2SrHG6MomaKmSEIvhHcyTRLQRSzdoZgN2Fkmwi1oT6Ja0ktmhdnZWo3J8SLVgQqF389pBZ0J+/kBP32v16148vr/fSFeLUVglYB49ot4jRolxALA7xBIhql/ujwOaIzG+5LrSz77iDK59o3wNq3Fa9NTINCE2hjC0bQNsk1RWG3/V5Sh2ezB5SaZvW8PqlUzrg2vxeBBy1MbsR/qg2CP8pqnaFdxVOjaqgYOn8qS2wcXj+wUk8PRuHlp8bPvk6tj96nDsOHLO5yAd2Lp0KfYqRhEjpWX980NsT9ejgdJOQIa1b+BbXzO/yI0VTw1EQpzLVwkzHNSvlojF4/v5OaN3Hg1PVcKpNq6Dtc8NxhNfbcKK3SfQrG4VpA9vjZqV42zNlAAgUeOgdhHBLbwrZFWvJDdFmTk2Zz3cG51e/lG6z3evushwjdjtR0L7Xf52TVv8aWALnwl2xyvDpOGtIw0GAPp1d5vWroxrOjb0+25kRJ2gyyrxuYjQsn6J6HVpUgMbDuQC8NotU+pU8RN0IHC09ccBzfHesj0Bx1bNHVmTRmLB1qO+1Yz0XNcp0LbaSWKqSWtaCyt2n0CyJo5Zne66PR7c3rMJDp3Ow3IlC7ZFPXsXuV3iYlx+D5i6QSQtRIJgkipKw41dk9H+stKv8lPa+tva2ci5/GKf6B7O9WZ0Xl4/9KUBteivqy3ZpRP0pU8OwLwtRzDUYpYW4yLUS0rE0HYNsGL3CV9lxIdsjNIbVEvE0bP5iI/RlooA3Mr/Rgutm9nQtT6mpIRYX2brVw9d4ZcjYBQ2aFRzaMkT/XG+oBij/vOLdH9sjMtv1mVkpjGaXWjHfmufHYxaVeLtVT+1bFHO3C8p3B/jgl8RIG2srToStrJ9Pj2sNX56oj/ev9Pf7KGdwuhDvrQVCK+2uVjBuEEt8NMT/f0iKVS7V7Fb4LUbOuDFa+VxwWVBRRr9liVv39IZ94VhUYjSuia08e65eSXZqY2V+Pa7rkgp3QcYUFqbd2qdKnhkoNxRpzLjgZ5Y9uQAACX3aDD+qR/H98MbN3X0G5Codm4XkWE9GyMb+lATB2KPVP9orPv7yu3oyZK6RwDQrG5VdGxUA3+4snTXlJFNXFsSoV61RNv3cdSN0GXV22KIkJZSCx/c1Q25FwtxXedkzFznDfdREz2eHNoKczYdNnUaNK9bNcC8ofUaV9Fl9Wmf/HadfTEuCvgMNaRLvanKwnHIlA/Bxkjr0f702gqGtasmhFSe1y5x5XDN9dZk5vZIrYUVTw20lYmpUi0xLiDEM0Yj6EbIRui7Xx3u5+8AgKl3p+G2ab9Kj3F7zya4vWcTpKT7F4zVLkgt47mRbTFtxT7TNmYYPaTMFmE3I+oEXRa4r05z9QsPt26Q5BNHWb1mlV+fGSzd3rxuFb8iRPrv9oKmRnlp7ocnhrZC3aQEXNPRO4vQrhjEVCxKWy5WO82XlaQtKyIxiDBKKAsGX7eV/3dNHI5h/1yON27uiJuVNYVlZbW1JtdxA1uga9MauKJ5bax7bgj2nwy0lRtx5EzFWlw86gRdlvwimy7u+/sIv6ebaqeWrUQky54DEJA0EhDOrzlWacq+VoqP8bMhhnOhDqZ8ISL88Oe+QYvV7Ef6YPH2Y37hrtUqld/tV1Fnhfp7JT7WhSWKWUfFKkrtSU05i7pJCQELephhVtysPAj2d4s6QZeN0GX2cTslXNOa1sTB08a1D/RvcZtkqIXzhqioNxfjJZTyB50a1/A50a9qWx+Lfj9mWNKhLKio15w6O/eY3JvhWP1KT+sGSdgRxsijUPj+kT6oXy24GWFEPWbaWhZquVJZLLCdzDm1xKlq1gCAWX/sjTXPBq7GoqJ/+qv2vruvaOpXQEfWtjTwCP3SRl2f83yB/WUHS0tFvebUqB+z/odS0dKIide3x4wHeuKFa70lO+yk+ofCazd0sFz2sHPjGgFJXVZEbITepmE19Lu8ri/284O7umH38fNBpeFrqRwfixVPDTT0Svu3jcHFQjc66n6sKrpiTtobLpwDnNKGvjEVmyFt6+PbDdm2S6aGgz8PaoGfd+XYLvEaLUy7Ow0bDuQarqwElH6ZPy3albDsOqk/f6Bn0Gs3qE7YcBMxQdemIl/Vtj6SEuPQtUnpVmqxa9fcMOEqfLoqC38wqcsB+Cc4hLoEmAz1SA3CmB3IVBxGdGiIzS8OLdcVdtJSapVpFE1ZUaNyPAZaFM+L9AAp1GUjy4KI2tBVjeyZGlihryxJiI2xVSiqrKapdZMS0LZhNTw1rOwXeWCik4q6XBoT3UTUhk6GRSLNkwIqOnExLsx77EppYaWygq08DON8IjpC19cr0PLajR3w4+/H0D2E2tCMP5tfHFphnWIMY8SOV4b5lSBmIizoaiaXzEtdp2oCVjw1MKiY0bLiiavKbqHi8oCn94wTSYyLCTmIwqlEVNAfG9ISMS4KWKBXJRyZZqXBbAbBMAwTbUTUhp6UGIdnRrQJacXu8kCtv93foOIawzBMNBF1maLRRNcmNStkqBfDMJcmXFuVYRjGIbCgMwzDOAQWdIZhGIfAgs4wDOMQWNAZhmEcAgs6wzCMQ2BBZxiGcQgs6AzDMA6BBZ1hGMYh2BJ0IhpGRDuJKJOI0iX7HyaiLUS0kYhWElHb8HeVYRiGMcNS0IkoBsAUAMMBtAVwm0SwZwghOgghOgN4A8DbYe8pwzAMY4qdEXoPAJlCiL1CiEIAMwFcp20ghDir+bMKwGUKGYZhyhs7xbmSARzU/H0IQE99IyJ6BMB4APEABoWldwzDMIxtwuYUFUJMEUI0B/A0gOdlbYjoQSLKIKKMnJyccH00wzAMA3uCng2gsebvRso2I2YCuF62QwgxVQiRJoRIq1uXa4wzDMOEEzuCvg5ASyJKJaJ4AGMAzNE2IKKWmj9HAtgdvi4yDMMwdrC0oQshioloHICFAGIATBdCbCOilwFkCCHmABhHREMAFAE4DeCesuw0wzAME4itFYuEEPMAzNNtm6B5/ViY+8UwDMMECWeKMgzDOAQWdIZhGIfAgs4wDOMQWNAZhmEcAgs6wzCMQ2BBZxiGcQgs6AzDMA6BBZ1hGMYhsKAzDMM4BBZ0hmEYh8CCzjAM4xBY0BmGYRwCCzrDMIxDYEFnGIZxCCzoDMMwDoEFnWEYxiGwoDMMwzgEFnSGYRiHwILOMAzjEFjQGYZhHAILOsMwjENgQWcYhnEILOgMwzAOgQWdYRjGIbCgMwzDOAQWdIZhGIfAgs4wDOMQWNAZhmEcAgs6wzCMQ2BBZxiGcQgs6AzDMA6BBZ1hGMYh2BJ0IhpGRDuJKJOI0iX7xxPR70S0mYh+IqKm4e8qwzAMY4aloBNRDIApAIYDaAvgNiJqq2u2AUCaEKIjgFkA3gh3RxmGYRhz7IzQewDIFELsFUIUApgJ4DptAyHEUiHEReXPXwE0Cm83GYZhGCvsCHoygIOavw8p24y4H8B82Q4iepCIMogoIycnx34vGYZhGEvC6hQlojsBpAF4U7ZfCDFVCJEmhEirW7duOD+aYRjmkifWRptsAI01fzdStvlBREMAPAegvxCiIDzdYxiGYexiZ4S+DkBLIkolongAYwDM0TYgoi4APgAwSghxPPzdZBiGYaywFHQhRDGAcQAWAtgO4CshxDYiepmIRinN3gRQFcDXRLSRiOYYHI5hGIYpI+yYXCCEmAdgnm7bBM3rIWHuF8MwDBMknCnKMAzjEFjQGYZhHAILOsMwjENgQWcYhnEILOgMwzAOgQWdYRjGIbCgMwzDOAQWdIZhGIfAgs4wDOMQWNAZhmEcAgs6wzCMQ2BBZxiGcQgs6AzDMA6BBZ1hGMYhsKAzDMM4BBZ0hmEYh8CCzjAM4xBY0BmGYRwCCzrDMIxDYEFnGIZxCCzoDMMwDoEFnWEYxiGwoDMMwzgEFnSGYRiHwILOMAzjEFjQGYZhHAILOsMwjENgQWcYhnEILOgMwzAOgQWdYRjGIbCgMwzDOAQWdIZhGIdgS9CJaBgR7SSiTCJKl+zvR0S/EVExEd0c/m4yDMMwVlgKOhHFAJgCYDiAtgBuI6K2umYHAIwFMCPcHWQYhmHsEWujTQ8AmUKIvQBARDMBXAfgd7WBECJL2ecpgz4yDMMwNrBjckkGcFDz9yFlW9AQ0YNElEFEGTk5OaEcgmEYhjGgXJ2iQoipQog0IURa3bp1y/OjGYZhHI8dQc8G0FjzdyNlG8MwDBNF2BH0dQBaElEqEcUDGANgTtl2i2EYhgkWS0EXQhQDGAdgIYDtAL4SQmwjopeJaBQAEFF3IjoEYDSAD4hoW1l2mmEYhgnETpQLhBDzAMzTbZugeb0OXlMMwzAMEyE4U5RhGMYhsKAzDMM4BBZ0hmEYh8CCzjAM4xBY0BmGYRwCCzrDMIxDYEFnGIZxCCzoDMMwDoEFnWEYxiGwoDMMwzgEFnSGYRiHwILOMAzjEFjQGYZhHAILOsMwjENgQWcYhnEILOgMwzAOgQWdYRjGIbCgMwzDOAQWdIZhGIfAgs4wDOMQWNAZhmEcAgs6wzCMQ2BBZxiGcQgs6AzDMA6BBZ1hGMYhsKAzDMM4BBZ0hmEYh8CCzjAM4xBY0BmGYRwCCzrDMIxDYEFnGIZxCCzoDMMwDsGWoBPRMCLaSUSZRJQu2Z9ARF8q+9cQUUq4O8owDMOYYynoRBQDYAqA4QDaAriNiNrqmt0P4LQQogWAdwC8Hu6OMgzDMObYGaH3AJAphNgrhCgEMBPAdbo21wH4VHk9C8BgIqLwdZNhGIaxItZGm2QABzV/HwLQ06iNEKKYiM4AqA3ghLYRET0I4EHlzwIi2hpKpx1AHei+m0uES/W8AT73S/Hcy+q8mxrtsCPoYUMIMRXAVAAgogwhRFp5fn60cKme+6V63gCf+6V47pE4bzsml2wAjTV/N1K2SdsQUSyA6gBOhqODDMMwjD3sCPo6AC2JKJWI4gGMATBH12YOgHuU1zcDWCKEEOHrJsMwDGOFpclFsYmPA7AQQAyA6UKIbUT0MoAMIcQcAB8B+C8RZQI4Ba/oWzG1FP2u6Fyq536pnjfA534pUu7nTTyQZhiGcQacKcowDOMQWNAZhmEcQkQE3aqUQEWEiLKIaAsRbSSiDGVbLSJaRES7lf9rKtuJiCYr57+ZiLpqjnOP0n43Ed1j9HmRhIimE9FxbR5BOM+ViLop32Wm8t6oSFIzOO8XiShb+d03EtEIzb5nlHPYSURXa7ZLr38l8GCNsv1LJQghKiCixkS0lIh+J6JtRPSYst3Rv7vJeUfn7y6EKNd/8DpW9wBoBiAewCYAbcu7H2VwXlkA6ui2vQEgXXmdDuB15fUIAPMBEIBeANYo22sB2Kv8X1N5XTPS5yY5134AugLYWhbnCmCt0paU9w6P9DmbnPeLAJ6UtG2rXNsJAFKVaz7G7PoH8BWAMcrr9wH8MdLnrDmfhgC6Kq+TAOxSztHRv7vJeUfl7x6JEbqdUgJOQVsS4VMA12u2fya8/AqgBhE1BHA1gEVCiFNCiNMAFgEYVt6dtkIIsRzeaCYtYTlXZV81IcSvwnuFf6Y5VkQxOG8jrgMwUwhRIITYByAT3mtfev0ro9FB8JbOAPy/w4gjhDgihPhNeX0OwHZ4M8Qd/bubnLcREf3dIyHoslICZl9QRUEA+JGI1pO3xAEA1BdCHFFeHwVQX3lt9B1U5O8mXOearLzWb49mxilmhemqyQHBn3dtALlCiGLd9qiDvNVUuwBYg0vod9edNxCFvzs7RcNHXyFEV3irUj5CRP20O5VRxyURI3opnSuA9wA0B9AZwBEAb0W2O2ULEVUF8A2Ax4UQZ7X7nPy7S847Kn/3SAi6nVICFQ4hRLby/3EA38E7xTqmTCWh/H9caW70HVTk7yZc55qtvNZvj0qEEMeEEG4hhAfANHh/dyD48z4Jr1kiVrc9aiCiOHhF7XMhxLfKZsf/7rLzjtbfPRKCbqeUQIWCiKoQUZL6GsBQAFvhXxLhHgCzlddzANytRAL0AnBGmbYuBDCUiGoqU7ihyraKQFjOVdl3loh6KfbFuzXHijpUMVO4Ad7fHfCe9xjyLv6SCqAlvE4/6fWvjG6Xwls6A/D/DiOO8lt8BGC7EOJtzS5H/+5G5x21v3t5eow1nuAR8HqL9wB4LhJ9CPP5NIPXa70JwDb1nOC1j/0EYDeAxQBqKdsJ3kVD9gDYAiBNc6z74HWkZAK4N9LnZnC+X8A7zSyC1+Z3fzjPFUCacoPsAfAfKBnNkf5ncN7/Vc5rM7w3c0NN++eUc9gJTcSG0fWvXEdrle/jawAJkT5nTd/6wmtO2Qxgo/JvhNN/d5PzjsrfnVP/GYZhHAI7RRmGYRwCCzrDMIxDYEFnGIZxCCzoDMMwDoEFnWEYxiGwoDMMwzgEFnSGYRiH8P8ESyCx36gtIwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wf350mCfqAPc",
        "outputId": "87849238-8730-461a-9672-afe3067dec95",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "def get_preds_as_nparray(ds_type) -> np.ndarray:\n",
        "    \"\"\"\n",
        "    the get_preds method does not yield the elements in order by default\n",
        "    we borrow the code from the RNNLearner to resort the elements into their correct order\n",
        "    \"\"\"\n",
        "    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n",
        "    sampler = [i for i in databunch.dl(ds_type).sampler]\n",
        "    reverse_sampler = np.argsort(sampler)\n",
        "    return preds[reverse_sampler, :]\n",
        "\n",
        "test_preds = get_preds_as_nparray(DatasetType.Test)\n"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bKOBN9v_qZz",
        "outputId": "8fee4d06-b1c8-472c-cc43-bf8e894c508e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "sample_submission = pd.DataFrame(np.argmax(test_preds,axis=1),columns=['label'])\n",
        "sample_submission.head()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label\n",
              "0      2\n",
              "1      0\n",
              "2      0\n",
              "3      0\n",
              "4      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hJfYZAeWAAWl",
        "outputId": "f8f91631-5be4-48ed-9528-f481e7dccb47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "sample_submission.shape"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(30000, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-77qKeWdADOk",
        "outputId": "95ddfe05-9804-4fd3-f5c0-46263c39a9cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "sample_submission.label.value_counts()  \n",
        "# O/p\n",
        "# 1    13896\n",
        "# 0    13798\n",
        "# 2     2306"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    13896\n",
              "0    13798\n",
              "2     2306\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2L_p3ddGAI3o"
      },
      "source": [
        "sample_submission.to_csv(\"roberta_base_1epochunfreeze_1epochfreeze_3_lr2e5.csv\",index=False)"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ra1M47MA2R7g"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}